Migrated to PG succesfully, I hope, haven't ran all the tests

This commit is contained in:
2025-09-03 14:51:29 +01:00
parent 4552b8c969
commit 6b719cb0ef
16 changed files with 118 additions and 111 deletions
+1 -1
View File
@@ -1,7 +1,7 @@
build = ./build/expensesweb build = ./build/expensesweb
registry = dir.esilva.org registry = dir.esilva.org
repo = esilva repo = esilva
tag = v0.0.80-alpha tag = v0.0.81-alpha
build_release_alpine: build_release_alpine:
rm -rf build && mkdir -p $(build) rm -rf build && mkdir -p $(build)
+31 -1
View File
@@ -1,6 +1,8 @@
# DOING # DOING
Migrating to postgres Migrating to postgres
Refactor how the models access the database (move the Tx creation to the models) Refactor how the models access the database (move the Tx creation to the models)
Refactor the Makefiles
Refactor the App start function
# TODO # TODO
- [ ] On the categories page, add a column of how many times a given category is - [ ] On the categories page, add a column of how many times a given category is
@@ -21,11 +23,39 @@ used
- [ ] Better UX when adding expenses on the payments and share values (default even split) - [ ] Better UX when adding expenses on the payments and share values (default even split)
- [ ] Verification popups for delete actions - [ ] Verification popups for delete actions
### Functionalities
#### Expenses
- Create
- View
- Update
- Delete
#### Categories
- View
- Create
- Delete
- Update
#### Stores
- View
- Create
- Delete
- Update
#### Types
- View
- Create
- Delete
- Update
### Stack ### Stack
- Go - Go
- golang-migrate - golang-migrate
- sqlc - sqlc
- sqlite ~- sqlite~
- postgresql
- python
- lua
### Patterns ### Patterns
- Service Pattern - Service Pattern
-67
View File
@@ -1,67 +0,0 @@
DROP TABLE IF EXISTS expenses;
CREATE TABLE expenses (
ExpID integer PRIMARY KEY AUTOINCREMENT,
Description text NOT NULL,
Value float NOT NULL DEFAULT 0,
StoreID int NOT NULL DEFAULT 0,
CategoryID int NOT NULL DEFAULT 0,
OwnerUserID int NOT NULL DEFAULT 0,
TypeID int NOT NULL DEFAULT 0,
ExpDate int NOT NULL DEFAULT -1,
CreationDate int NOT NULL DEFAULT -1,
FOREIGN KEY(StoreID) REFERENCES stores(StoreID) ON DELETE RESTRICT,
FOREIGN KEY(CategoryID) REFERENCES categories(CategoryID) ON DELETE RESTRICT,
FOREIGN KEY(OwnerUserID) REFERENCES users(UserID) ON DELETE RESTRICT,
FOREIGN KEY(TypeID) REFERENCES expenseTypes(TypeID) ON DELETE RESTRICT
);
DROP TABLE IF EXISTS stores;
CREATE TABLE stores(
StoreID integer PRIMARY KEY AUTOINCREMENT,
StoreName text NOT NULL
);
-- Create a default store for NoStore
INSERT INTO stores (StoreID, StoreName) VALUES(0, 'No Store');
DROP TABLE IF EXISTS categories;
CREATE TABLE categories(
CategoryID integer PRIMARY KEY AUTOINCREMENT,
CategoryName text NOT NULL
);
-- Create a default category for NoCategory
INSERT INTO categories (CategoryID, CategoryName) VALUES(0, 'No Category');
DROP TABLE IF EXISTS expenseTypes;
CREATE TABLE expenseTypes(
TypeID integer PRIMARY KEY AUTOINCREMENT,
TypeName text NOT NULL
);
-- Create a default category for NoCategory
INSERT INTO expenseTypes (TypeID, TypeName) VALUES(0, 'expense');
DROP TABLE IF EXISTS users;
CREATE TABLE users(
UserID integer PRIMARY KEY AUTOINCREMENT,
UserName text NOT NULL,
UserPass text NOT NULL
);
DROP TABLE IF EXISTS expensesShares;
CREATE TABLE expensesShares(
ExpShareID integer PRIMARY KEY AUTOINCREMENT,
ExpID int NOT NULL,
UserID int NOT NULL,
Share float NOT NULL DEFAULT 0.5,
FOREIGN KEY(ExpID) REFERENCES expenses(ExpID) ON DELETE RESTRICT,
FOREIGN KEY(UserID) REFERENCES users(UserID) ON DELETE RESTRICT
);
DROP TABLE IF EXISTS expensesPayments;
CREATE TABLE expensesPayments(
ExpPaymID integer PRIMARY KEY AUTOINCREMENT,
ExpID int NOT NULL,
UserID int NOT NULL,
Payed float NOT NULL DEFAULT 0,
FOREIGN KEY(ExpID) REFERENCES expenses(ExpID) ON DELETE RESTRICT,
FOREIGN KEY(UserID) REFERENCES users(UserID) ON DELETE RESTRICT
);
+14 -6
View File
@@ -2,7 +2,7 @@
import psycopg2 import psycopg2
from psycopg2 import Error from psycopg2 import Error
from datetime import datetime, timedelta from datetime import datetime
class DB(): class DB():
@@ -86,12 +86,24 @@ def mig_data(db, m):
fields = ','.join([f"\"{h}\"" for h in m["headers"]]) fields = ','.join([f"\"{h}\"" for h in m["headers"]])
query = f"INSERT INTO \"{m["table"]}\" ({fields}) VALUES({values})" query = f"INSERT INTO \"{m["table"]}\" ({fields}) VALUES({values})"
lastId = -1
try: try:
data = get_data(m["csvFile"]) data = get_data(m["csvFile"])
insert_data(db, query, data) insert_data(db, query, data)
lastId = data[0]
except (Exception, Error) as error: except (Exception, Error) as error:
raise error raise error
if lastId != -1:
if m["table"] == "schema_migrations":
return
try:
db.runQuery(
f"SELECT setval(pg_get_serial_sequence('\"{m["table"]}\"', '{m["headers"][0]}'),"
f"(SELECT MAX(\"{m["headers"][0]}\") FROM \"{m["table"]}\"))", []
)
except (Exception, Error) as error:
raise error
if __name__ == "__main__": if __name__ == "__main__":
if False: if False:
@@ -105,11 +117,6 @@ if __name__ == "__main__":
exit(0) exit(0)
migs = [ migs = [
{
"table": "schema_migrations",
"csvFile": "./schema_migrations.csv",
"headers": ["version", "dirty"],
},
{ {
"table": "categories", "table": "categories",
"csvFile": "./categories.csv", "csvFile": "./categories.csv",
@@ -157,3 +164,4 @@ if __name__ == "__main__":
mig_data(dbObj, m) mig_data(dbObj, m)
except (Exception, Error) as error: except (Exception, Error) as error:
print("failed to migrate data:", error) print("failed to migrate data:", error)
exit(1)
+15 -2
View File
@@ -6,6 +6,7 @@ import (
"github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5"
lua "github.com/yuin/gopher-lua" lua "github.com/yuin/gopher-lua"
"log" "log"
"strings"
) )
type ExpensesApp struct { type ExpensesApp struct {
@@ -37,12 +38,24 @@ func StartLuaVM() (*lua.LState, error) {
return L, nil return L, nil
} }
func getPgConnString(cfg *config.Configuration) string {
var s strings.Builder
s.WriteString("port=" + cfg.PgCfg.Port + " ")
s.WriteString("host=" + cfg.PgCfg.Host + " ")
s.WriteString("user=" + cfg.PgCfg.User + " ")
s.WriteString("dbname=" + cfg.PgCfg.DB + " ")
s.WriteString("password=" + cfg.PgCfg.Pass)
return s.String()
}
func StartApp() error { func StartApp() error {
config.SetConfig("./config.yaml") config.SetConfig("./config.yaml")
// cfg := config.GetInstance() cfg := config.GetInstance()
ctx := context.Background() ctx := context.Background()
pgStr := "port=5431 host=127.0.0.1 user=expuser dbname=expdb password=exppass" pgStr := getPgConnString(cfg)
conn, err := pgx.Connect(ctx, pgStr) conn, err := pgx.Connect(ctx, pgStr)
if err != nil { if err != nil {
log.Fatalf("Failed to open migration DB: %v", err) log.Fatalf("Failed to open migration DB: %v", err)
+13 -7
View File
@@ -7,11 +7,11 @@ SELECT
sqlc.embed(types) sqlc.embed(types)
FROM expenses FROM expenses
JOIN JOIN
Stores ON stores."StoreID" = expenses."StoreID" stores ON stores."StoreID" = expenses."StoreID"
JOIN JOIN
Categories ON categories."CategoryID" = expenses."CategoryID" categories ON categories."CategoryID" = expenses."CategoryID"
JOIN JOIN
Users ON users."UserID" = expenses."OwnerUserID" users ON users."UserID" = expenses."OwnerUserID"
JOIN JOIN
"expenseTypes" as types ON types."TypeID" = expenses."TypeID" "expenseTypes" as types ON types."TypeID" = expenses."TypeID"
WHERE WHERE
@@ -28,11 +28,11 @@ SELECT
sqlc.embed(types) sqlc.embed(types)
FROM expenses FROM expenses
JOIN JOIN
Stores ON stores."StoreID" = expenses."StoreID" stores ON stores."StoreID" = expenses."StoreID"
JOIN JOIN
Categories ON categories."CategoryID" = expenses."CategoryID" categories ON categories."CategoryID" = expenses."CategoryID"
JOIN JOIN
Users ON "UserID" = "OwnerUserID" users ON "UserID" = "OwnerUserID"
JOIN JOIN
"expenseTypes" as types ON types."TypeID" = expenses."TypeID" "expenseTypes" as types ON types."TypeID" = expenses."TypeID"
WHERE WHERE
@@ -55,7 +55,13 @@ VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
RETURNING "ExpID"; RETURNING "ExpID";
-- name: DeleteExpense :execresult -- name: DeleteExpense :execresult
DELETE FROM expenses WHERE "ExpID" = $1; WITH delPayments AS (
DELETE FROM "expensesPayments" WHERE "ExpID" = $1
),
delShares AS (
DELETE FROM "expensesShares" WHERE "ExpID" = $1
)
DELETE FROM expenses WHERE expenses."ExpID" = $1;
-- name: UpdateExpense :execresult -- name: UpdateExpense :execresult
UPDATE expenses UPDATE expenses
+2 -2
View File
@@ -4,7 +4,7 @@ SELECT
FROM FROM
"expensesPayments" as payments "expensesPayments" as payments
JOIN JOIN
users as users ON users.UserID = payments.UserID users as users ON users."UserID" = payments."UserID"
WHERE "ExpID" = $1; WHERE "ExpID" = $1;
-- name: GetExpensePaymentByUser :one -- name: GetExpensePaymentByUser :one
@@ -13,7 +13,7 @@ SELECT
FROM FROM
"expensesPayments" as payments "expensesPayments" as payments
JOIN JOIN
users as users ON users.UserID = payments.UserID users as users ON users."UserID" = payments."UserID"
WHERE WHERE
"ExpID" = $1 AND users."UserID" = $2; "ExpID" = $1 AND users."UserID" = $2;
+1 -1
View File
@@ -4,7 +4,7 @@ SELECT
FROM FROM
"expensesShares" as shares "expensesShares" as shares
JOIN JOIN
users as users ON users.UserID = shares.UserID users as users ON users."UserID" = shares."UserID"
WHERE "ExpID" = $1; WHERE "ExpID" = $1;
-- name: InsertShare :execresult -- name: InsertShare :execresult
+13 -7
View File
@@ -13,7 +13,13 @@ import (
) )
const deleteExpense = `-- name: DeleteExpense :execresult const deleteExpense = `-- name: DeleteExpense :execresult
DELETE FROM expenses WHERE "ExpID" = $1 WITH delPayments AS (
DELETE FROM "expensesPayments" WHERE "ExpID" = $1
),
delShares AS (
DELETE FROM "expensesShares" WHERE "ExpID" = $1
)
DELETE FROM expenses WHERE expenses."ExpID" = $1
` `
func (q *Queries) DeleteExpense(ctx context.Context, expid int32) (pgconn.CommandTag, error) { func (q *Queries) DeleteExpense(ctx context.Context, expid int32) (pgconn.CommandTag, error) {
@@ -29,11 +35,11 @@ SELECT
types."TypeID", types."TypeName" types."TypeID", types."TypeName"
FROM expenses FROM expenses
JOIN JOIN
Stores ON stores."StoreID" = expenses."StoreID" stores ON stores."StoreID" = expenses."StoreID"
JOIN JOIN
Categories ON categories."CategoryID" = expenses."CategoryID" categories ON categories."CategoryID" = expenses."CategoryID"
JOIN JOIN
Users ON "UserID" = "OwnerUserID" users ON "UserID" = "OwnerUserID"
JOIN JOIN
"expenseTypes" as types ON types."TypeID" = expenses."TypeID" "expenseTypes" as types ON types."TypeID" = expenses."TypeID"
WHERE WHERE
@@ -85,11 +91,11 @@ SELECT
types."TypeID", types."TypeName" types."TypeID", types."TypeName"
FROM expenses FROM expenses
JOIN JOIN
Stores ON stores."StoreID" = expenses."StoreID" stores ON stores."StoreID" = expenses."StoreID"
JOIN JOIN
Categories ON categories."CategoryID" = expenses."CategoryID" categories ON categories."CategoryID" = expenses."CategoryID"
JOIN JOIN
Users ON users."UserID" = expenses."OwnerUserID" users ON users."UserID" = expenses."OwnerUserID"
JOIN JOIN
"expenseTypes" as types ON types."TypeID" = expenses."TypeID" "expenseTypes" as types ON types."TypeID" = expenses."TypeID"
WHERE WHERE
+2 -2
View File
@@ -26,7 +26,7 @@ SELECT
FROM FROM
"expensesPayments" as payments "expensesPayments" as payments
JOIN JOIN
users as users ON users.UserID = payments.UserID users as users ON users."UserID" = payments."UserID"
WHERE WHERE
"ExpID" = $1 AND users."UserID" = $2 "ExpID" = $1 AND users."UserID" = $2
` `
@@ -62,7 +62,7 @@ SELECT
FROM FROM
"expensesPayments" as payments "expensesPayments" as payments
JOIN JOIN
users as users ON users.UserID = payments.UserID users as users ON users."UserID" = payments."UserID"
WHERE "ExpID" = $1 WHERE "ExpID" = $1
` `
+1 -1
View File
@@ -26,7 +26,7 @@ SELECT
FROM FROM
"expensesShares" as shares "expensesShares" as shares
JOIN JOIN
users as users ON users.UserID = shares.UserID users as users ON users."UserID" = shares."UserID"
WHERE "ExpID" = $1 WHERE "ExpID" = $1
` `
+7 -7
View File
@@ -3,28 +3,28 @@
-- This should only be a workaround :c -- This should only be a workaround :c
CREATE TABLE categories ( CREATE TABLE categories (
"CategoryID" SERIAL PRIMARY KEY, "CategoryID" INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
"CategoryName" TEXT NOT NULL "CategoryName" TEXT NOT NULL
); );
CREATE TABLE "expenseTypes" ( CREATE TABLE "expenseTypes" (
"TypeID" SERIAL PRIMARY KEY, "TypeID" INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
"TypeName" TEXT NOT NULL "TypeName" TEXT NOT NULL
); );
CREATE TABLE stores ( CREATE TABLE stores (
"StoreID" SERIAL PRIMARY KEY, "StoreID" INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
"StoreName" TEXT NOT NULL "StoreName" TEXT NOT NULL
); );
CREATE TABLE users ( CREATE TABLE users (
"UserID" SERIAL PRIMARY KEY, "UserID" INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
"UserName" TEXT NOT NULL, "UserName" TEXT NOT NULL,
"UserPass" TEXT NOT NULL "UserPass" TEXT NOT NULL
); );
CREATE TABLE expenses ( CREATE TABLE expenses (
"ExpID" SERIAL PRIMARY KEY, "ExpID" INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
"Description" TEXT NOT NULL, "Description" TEXT NOT NULL,
"Value" NUMERIC(12,2) NOT NULL DEFAULT 0, "Value" NUMERIC(12,2) NOT NULL DEFAULT 0,
"StoreID" INT NOT NULL DEFAULT 0, "StoreID" INT NOT NULL DEFAULT 0,
@@ -42,7 +42,7 @@ CREATE TABLE expenses (
); );
CREATE TABLE "expensesPayments" ( CREATE TABLE "expensesPayments" (
"ExpPaymID" SERIAL PRIMARY KEY, "ExpPaymID" INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
"ExpID" INT NOT NULL, "ExpID" INT NOT NULL,
"UserID" INT NOT NULL, "UserID" INT NOT NULL,
"Payed" NUMERIC(12,2) NOT NULL DEFAULT 0, "Payed" NUMERIC(12,2) NOT NULL DEFAULT 0,
@@ -51,7 +51,7 @@ CREATE TABLE "expensesPayments" (
); );
CREATE TABLE "expensesShares" ( CREATE TABLE "expensesShares" (
"ExpShareID" SERIAL PRIMARY KEY, "ExpShareID" INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
"ExpID" INT NOT NULL, "ExpID" INT NOT NULL,
"UserID" INT NOT NULL, "UserID" INT NOT NULL,
"Share" NUMERIC(6,2) NOT NULL DEFAULT 0.5, "Share" NUMERIC(6,2) NOT NULL DEFAULT 0.5,
+5 -4
View File
@@ -1,16 +1,13 @@
package pages package pages
import ( import (
"database/sql"
"encoding/json" "encoding/json"
exp "expenses/expenses" exp "expenses/expenses"
experr "expenses/expenses/errors" experr "expenses/expenses/errors"
mod "expenses/expenses/models" mod "expenses/expenses/models"
"fmt" "fmt"
"log" "log"
"net/http" "net/http"
"github.com/gin-gonic/gin" "github.com/gin-gonic/gin"
) )
@@ -54,19 +51,21 @@ func expensePage(c *gin.Context) {
} }
expense, err := exp.App.GetExpense(expenseID) expense, err := exp.App.GetExpense(expenseID)
if err == sql.ErrNoRows { if err != nil {
NotFoundView(c, "didn't find requested expense") NotFoundView(c, "didn't find requested expense")
return return
} }
err = exp.App.LoadExpenseShares(&expense) err = exp.App.LoadExpenseShares(&expense)
if err != nil { if err != nil {
log.Println("error:", err)
ServerErrorView(c, "failed to fetch shares") ServerErrorView(c, "failed to fetch shares")
return return
} }
err = exp.App.LoadExpensePayments(&expense) err = exp.App.LoadExpensePayments(&expense)
if err != nil { if err != nil {
log.Println("error:", err)
ServerErrorView(c, "failed to fetch payments") ServerErrorView(c, "failed to fetch payments")
return return
} }
@@ -211,12 +210,14 @@ func deleteExpense(c *gin.Context) {
err = exp.App.DeleteExpense(expenseID) err = exp.App.DeleteExpense(expenseID)
if err == experr.ErrNotFound { if err == experr.ErrNotFound {
log.Println("error:", err)
errMsg := fmt.Sprintf("category %d not found", expenseID) errMsg := fmt.Sprintf("category %d not found", expenseID)
c.String(http.StatusNotFound, errMsg) c.String(http.StatusNotFound, errMsg)
return return
} }
if err != nil { if err != nil {
log.Println("error:", err)
errMsg := fmt.Sprintf("error deleting category %d", expenseID) errMsg := fmt.Sprintf("error deleting category %d", expenseID)
c.String(http.StatusInternalServerError, errMsg) c.String(http.StatusInternalServerError, errMsg)
return return
+1
View File
@@ -27,6 +27,7 @@ EXPOSE 5431
# Copies the start script # Copies the start script
COPY ./start.sh /start.sh COPY ./start.sh /start.sh
COPY ./healthcheck.sh /healthcheck.sh
USER $UNAME USER $UNAME
STOPSIGNAL SIGTERM STOPSIGNAL SIGTERM
ENTRYPOINT [ "/start.sh" ] ENTRYPOINT [ "/start.sh" ]
+8 -3
View File
@@ -1,5 +1,7 @@
service = expenses-pgdb service = expenses-pgdb
version = v0.0.81-alpha registry = dir.esilva.org
repo = esilva
tag = v0.0.81-alpha
image = $(service) image = $(service)
container = $(image)-container container = $(image)-container
@@ -12,7 +14,10 @@ clean:
build: build:
cp ../expenses/db/schema.sql . cp ../expenses/db/schema.sql .
docker buildx build \ docker buildx build \
-t $(image) . -t $(registry)/$(repo)/expensesdb:$(tag) .
push_image:
docker push $(registry)/$(repo)/expensesdb:$(tag)
attach: attach:
docker exec -it $(container) zsh docker exec -it $(container) zsh
@@ -26,6 +31,6 @@ remove-dev-container:
test: remove-dev-container test: remove-dev-container
docker run -p 5431:5432 \ docker run -p 5431:5432 \
--name $(container)-dev \ --name $(container)-dev \
$$(cat .env | sed 's/^/-e /') \ --env-file .env \
-v ./pgsql-data-dev/:/var/lib/postgresql/data/ \ -v ./pgsql-data-dev/:/var/lib/postgresql/data/ \
-it $(image) -it $(image)
+4
View File
@@ -0,0 +1,4 @@
#!/bin/bash
set -e
pg_isready -U "$ExpDB_USER" -d "$ExpDB_NAME"