Migrated to PG succesfully, I hope, haven't ran all the tests

This commit is contained in:
2025-09-03 14:51:29 +01:00
parent 4552b8c969
commit 6b719cb0ef
16 changed files with 118 additions and 111 deletions
+1 -1
View File
@@ -1,7 +1,7 @@
build = ./build/expensesweb
registry = dir.esilva.org
repo = esilva
tag = v0.0.80-alpha
tag = v0.0.81-alpha
build_release_alpine:
rm -rf build && mkdir -p $(build)
+31 -1
View File
@@ -1,6 +1,8 @@
# DOING
Migrating to postgres
Refactor how the models access the database (move the Tx creation to the models)
Refactor the Makefiles
Refactor the App start function
# TODO
- [ ] On the categories page, add a column of how many times a given category is
@@ -21,11 +23,39 @@ used
- [ ] Better UX when adding expenses on the payments and share values (default even split)
- [ ] Verification popups for delete actions
### Functionalities
#### Expenses
- Create
- View
- Update
- Delete
#### Categories
- View
- Create
- Delete
- Update
#### Stores
- View
- Create
- Delete
- Update
#### Types
- View
- Create
- Delete
- Update
### Stack
- Go
- golang-migrate
- sqlc
- sqlite
~- sqlite~
- postgresql
- python
- lua
### Patterns
- Service Pattern
-67
View File
@@ -1,67 +0,0 @@
DROP TABLE IF EXISTS expenses;
CREATE TABLE expenses (
ExpID integer PRIMARY KEY AUTOINCREMENT,
Description text NOT NULL,
Value float NOT NULL DEFAULT 0,
StoreID int NOT NULL DEFAULT 0,
CategoryID int NOT NULL DEFAULT 0,
OwnerUserID int NOT NULL DEFAULT 0,
TypeID int NOT NULL DEFAULT 0,
ExpDate int NOT NULL DEFAULT -1,
CreationDate int NOT NULL DEFAULT -1,
FOREIGN KEY(StoreID) REFERENCES stores(StoreID) ON DELETE RESTRICT,
FOREIGN KEY(CategoryID) REFERENCES categories(CategoryID) ON DELETE RESTRICT,
FOREIGN KEY(OwnerUserID) REFERENCES users(UserID) ON DELETE RESTRICT,
FOREIGN KEY(TypeID) REFERENCES expenseTypes(TypeID) ON DELETE RESTRICT
);
DROP TABLE IF EXISTS stores;
CREATE TABLE stores(
StoreID integer PRIMARY KEY AUTOINCREMENT,
StoreName text NOT NULL
);
-- Create a default store for NoStore
INSERT INTO stores (StoreID, StoreName) VALUES(0, 'No Store');
DROP TABLE IF EXISTS categories;
CREATE TABLE categories(
CategoryID integer PRIMARY KEY AUTOINCREMENT,
CategoryName text NOT NULL
);
-- Create a default category for NoCategory
INSERT INTO categories (CategoryID, CategoryName) VALUES(0, 'No Category');
DROP TABLE IF EXISTS expenseTypes;
CREATE TABLE expenseTypes(
TypeID integer PRIMARY KEY AUTOINCREMENT,
TypeName text NOT NULL
);
-- Create a default category for NoCategory
INSERT INTO expenseTypes (TypeID, TypeName) VALUES(0, 'expense');
DROP TABLE IF EXISTS users;
CREATE TABLE users(
UserID integer PRIMARY KEY AUTOINCREMENT,
UserName text NOT NULL,
UserPass text NOT NULL
);
DROP TABLE IF EXISTS expensesShares;
CREATE TABLE expensesShares(
ExpShareID integer PRIMARY KEY AUTOINCREMENT,
ExpID int NOT NULL,
UserID int NOT NULL,
Share float NOT NULL DEFAULT 0.5,
FOREIGN KEY(ExpID) REFERENCES expenses(ExpID) ON DELETE RESTRICT,
FOREIGN KEY(UserID) REFERENCES users(UserID) ON DELETE RESTRICT
);
DROP TABLE IF EXISTS expensesPayments;
CREATE TABLE expensesPayments(
ExpPaymID integer PRIMARY KEY AUTOINCREMENT,
ExpID int NOT NULL,
UserID int NOT NULL,
Payed float NOT NULL DEFAULT 0,
FOREIGN KEY(ExpID) REFERENCES expenses(ExpID) ON DELETE RESTRICT,
FOREIGN KEY(UserID) REFERENCES users(UserID) ON DELETE RESTRICT
);
+14 -6
View File
@@ -2,7 +2,7 @@
import psycopg2
from psycopg2 import Error
from datetime import datetime, timedelta
from datetime import datetime
class DB():
@@ -86,12 +86,24 @@ def mig_data(db, m):
fields = ','.join([f"\"{h}\"" for h in m["headers"]])
query = f"INSERT INTO \"{m["table"]}\" ({fields}) VALUES({values})"
lastId = -1
try:
data = get_data(m["csvFile"])
insert_data(db, query, data)
lastId = data[0]
except (Exception, Error) as error:
raise error
if lastId != -1:
if m["table"] == "schema_migrations":
return
try:
db.runQuery(
f"SELECT setval(pg_get_serial_sequence('\"{m["table"]}\"', '{m["headers"][0]}'),"
f"(SELECT MAX(\"{m["headers"][0]}\") FROM \"{m["table"]}\"))", []
)
except (Exception, Error) as error:
raise error
if __name__ == "__main__":
if False:
@@ -105,11 +117,6 @@ if __name__ == "__main__":
exit(0)
migs = [
{
"table": "schema_migrations",
"csvFile": "./schema_migrations.csv",
"headers": ["version", "dirty"],
},
{
"table": "categories",
"csvFile": "./categories.csv",
@@ -157,3 +164,4 @@ if __name__ == "__main__":
mig_data(dbObj, m)
except (Exception, Error) as error:
print("failed to migrate data:", error)
exit(1)
+15 -2
View File
@@ -6,6 +6,7 @@ import (
"github.com/jackc/pgx/v5"
lua "github.com/yuin/gopher-lua"
"log"
"strings"
)
type ExpensesApp struct {
@@ -37,12 +38,24 @@ func StartLuaVM() (*lua.LState, error) {
return L, nil
}
func getPgConnString(cfg *config.Configuration) string {
var s strings.Builder
s.WriteString("port=" + cfg.PgCfg.Port + " ")
s.WriteString("host=" + cfg.PgCfg.Host + " ")
s.WriteString("user=" + cfg.PgCfg.User + " ")
s.WriteString("dbname=" + cfg.PgCfg.DB + " ")
s.WriteString("password=" + cfg.PgCfg.Pass)
return s.String()
}
func StartApp() error {
config.SetConfig("./config.yaml")
// cfg := config.GetInstance()
cfg := config.GetInstance()
ctx := context.Background()
pgStr := "port=5431 host=127.0.0.1 user=expuser dbname=expdb password=exppass"
pgStr := getPgConnString(cfg)
conn, err := pgx.Connect(ctx, pgStr)
if err != nil {
log.Fatalf("Failed to open migration DB: %v", err)
+13 -7
View File
@@ -7,11 +7,11 @@ SELECT
sqlc.embed(types)
FROM expenses
JOIN
Stores ON stores."StoreID" = expenses."StoreID"
stores ON stores."StoreID" = expenses."StoreID"
JOIN
Categories ON categories."CategoryID" = expenses."CategoryID"
categories ON categories."CategoryID" = expenses."CategoryID"
JOIN
Users ON users."UserID" = expenses."OwnerUserID"
users ON users."UserID" = expenses."OwnerUserID"
JOIN
"expenseTypes" as types ON types."TypeID" = expenses."TypeID"
WHERE
@@ -28,11 +28,11 @@ SELECT
sqlc.embed(types)
FROM expenses
JOIN
Stores ON stores."StoreID" = expenses."StoreID"
stores ON stores."StoreID" = expenses."StoreID"
JOIN
Categories ON categories."CategoryID" = expenses."CategoryID"
categories ON categories."CategoryID" = expenses."CategoryID"
JOIN
Users ON "UserID" = "OwnerUserID"
users ON "UserID" = "OwnerUserID"
JOIN
"expenseTypes" as types ON types."TypeID" = expenses."TypeID"
WHERE
@@ -55,7 +55,13 @@ VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
RETURNING "ExpID";
-- name: DeleteExpense :execresult
DELETE FROM expenses WHERE "ExpID" = $1;
WITH delPayments AS (
DELETE FROM "expensesPayments" WHERE "ExpID" = $1
),
delShares AS (
DELETE FROM "expensesShares" WHERE "ExpID" = $1
)
DELETE FROM expenses WHERE expenses."ExpID" = $1;
-- name: UpdateExpense :execresult
UPDATE expenses
+2 -2
View File
@@ -4,7 +4,7 @@ SELECT
FROM
"expensesPayments" as payments
JOIN
users as users ON users.UserID = payments.UserID
users as users ON users."UserID" = payments."UserID"
WHERE "ExpID" = $1;
-- name: GetExpensePaymentByUser :one
@@ -13,7 +13,7 @@ SELECT
FROM
"expensesPayments" as payments
JOIN
users as users ON users.UserID = payments.UserID
users as users ON users."UserID" = payments."UserID"
WHERE
"ExpID" = $1 AND users."UserID" = $2;
+1 -1
View File
@@ -4,7 +4,7 @@ SELECT
FROM
"expensesShares" as shares
JOIN
users as users ON users.UserID = shares.UserID
users as users ON users."UserID" = shares."UserID"
WHERE "ExpID" = $1;
-- name: InsertShare :execresult
+13 -7
View File
@@ -13,7 +13,13 @@ import (
)
const deleteExpense = `-- name: DeleteExpense :execresult
DELETE FROM expenses WHERE "ExpID" = $1
WITH delPayments AS (
DELETE FROM "expensesPayments" WHERE "ExpID" = $1
),
delShares AS (
DELETE FROM "expensesShares" WHERE "ExpID" = $1
)
DELETE FROM expenses WHERE expenses."ExpID" = $1
`
func (q *Queries) DeleteExpense(ctx context.Context, expid int32) (pgconn.CommandTag, error) {
@@ -29,11 +35,11 @@ SELECT
types."TypeID", types."TypeName"
FROM expenses
JOIN
Stores ON stores."StoreID" = expenses."StoreID"
stores ON stores."StoreID" = expenses."StoreID"
JOIN
Categories ON categories."CategoryID" = expenses."CategoryID"
categories ON categories."CategoryID" = expenses."CategoryID"
JOIN
Users ON "UserID" = "OwnerUserID"
users ON "UserID" = "OwnerUserID"
JOIN
"expenseTypes" as types ON types."TypeID" = expenses."TypeID"
WHERE
@@ -85,11 +91,11 @@ SELECT
types."TypeID", types."TypeName"
FROM expenses
JOIN
Stores ON stores."StoreID" = expenses."StoreID"
stores ON stores."StoreID" = expenses."StoreID"
JOIN
Categories ON categories."CategoryID" = expenses."CategoryID"
categories ON categories."CategoryID" = expenses."CategoryID"
JOIN
Users ON users."UserID" = expenses."OwnerUserID"
users ON users."UserID" = expenses."OwnerUserID"
JOIN
"expenseTypes" as types ON types."TypeID" = expenses."TypeID"
WHERE
+2 -2
View File
@@ -26,7 +26,7 @@ SELECT
FROM
"expensesPayments" as payments
JOIN
users as users ON users.UserID = payments.UserID
users as users ON users."UserID" = payments."UserID"
WHERE
"ExpID" = $1 AND users."UserID" = $2
`
@@ -62,7 +62,7 @@ SELECT
FROM
"expensesPayments" as payments
JOIN
users as users ON users.UserID = payments.UserID
users as users ON users."UserID" = payments."UserID"
WHERE "ExpID" = $1
`
+1 -1
View File
@@ -26,7 +26,7 @@ SELECT
FROM
"expensesShares" as shares
JOIN
users as users ON users.UserID = shares.UserID
users as users ON users."UserID" = shares."UserID"
WHERE "ExpID" = $1
`
+7 -7
View File
@@ -3,28 +3,28 @@
-- This should only be a workaround :c
CREATE TABLE categories (
"CategoryID" SERIAL PRIMARY KEY,
"CategoryID" INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
"CategoryName" TEXT NOT NULL
);
CREATE TABLE "expenseTypes" (
"TypeID" SERIAL PRIMARY KEY,
"TypeID" INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
"TypeName" TEXT NOT NULL
);
CREATE TABLE stores (
"StoreID" SERIAL PRIMARY KEY,
"StoreID" INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
"StoreName" TEXT NOT NULL
);
CREATE TABLE users (
"UserID" SERIAL PRIMARY KEY,
"UserID" INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
"UserName" TEXT NOT NULL,
"UserPass" TEXT NOT NULL
);
CREATE TABLE expenses (
"ExpID" SERIAL PRIMARY KEY,
"ExpID" INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
"Description" TEXT NOT NULL,
"Value" NUMERIC(12,2) NOT NULL DEFAULT 0,
"StoreID" INT NOT NULL DEFAULT 0,
@@ -42,7 +42,7 @@ CREATE TABLE expenses (
);
CREATE TABLE "expensesPayments" (
"ExpPaymID" SERIAL PRIMARY KEY,
"ExpPaymID" INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
"ExpID" INT NOT NULL,
"UserID" INT NOT NULL,
"Payed" NUMERIC(12,2) NOT NULL DEFAULT 0,
@@ -51,7 +51,7 @@ CREATE TABLE "expensesPayments" (
);
CREATE TABLE "expensesShares" (
"ExpShareID" SERIAL PRIMARY KEY,
"ExpShareID" INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
"ExpID" INT NOT NULL,
"UserID" INT NOT NULL,
"Share" NUMERIC(6,2) NOT NULL DEFAULT 0.5,
+5 -4
View File
@@ -1,16 +1,13 @@
package pages
import (
"database/sql"
"encoding/json"
exp "expenses/expenses"
experr "expenses/expenses/errors"
mod "expenses/expenses/models"
"fmt"
"log"
"net/http"
"github.com/gin-gonic/gin"
)
@@ -54,19 +51,21 @@ func expensePage(c *gin.Context) {
}
expense, err := exp.App.GetExpense(expenseID)
if err == sql.ErrNoRows {
if err != nil {
NotFoundView(c, "didn't find requested expense")
return
}
err = exp.App.LoadExpenseShares(&expense)
if err != nil {
log.Println("error:", err)
ServerErrorView(c, "failed to fetch shares")
return
}
err = exp.App.LoadExpensePayments(&expense)
if err != nil {
log.Println("error:", err)
ServerErrorView(c, "failed to fetch payments")
return
}
@@ -211,12 +210,14 @@ func deleteExpense(c *gin.Context) {
err = exp.App.DeleteExpense(expenseID)
if err == experr.ErrNotFound {
log.Println("error:", err)
errMsg := fmt.Sprintf("category %d not found", expenseID)
c.String(http.StatusNotFound, errMsg)
return
}
if err != nil {
log.Println("error:", err)
errMsg := fmt.Sprintf("error deleting category %d", expenseID)
c.String(http.StatusInternalServerError, errMsg)
return
+1
View File
@@ -27,6 +27,7 @@ EXPOSE 5431
# Copies the start script
COPY ./start.sh /start.sh
COPY ./healthcheck.sh /healthcheck.sh
USER $UNAME
STOPSIGNAL SIGTERM
ENTRYPOINT [ "/start.sh" ]
+8 -3
View File
@@ -1,5 +1,7 @@
service = expenses-pgdb
version = v0.0.81-alpha
registry = dir.esilva.org
repo = esilva
tag = v0.0.81-alpha
image = $(service)
container = $(image)-container
@@ -12,7 +14,10 @@ clean:
build:
cp ../expenses/db/schema.sql .
docker buildx build \
-t $(image) .
-t $(registry)/$(repo)/expensesdb:$(tag) .
push_image:
docker push $(registry)/$(repo)/expensesdb:$(tag)
attach:
docker exec -it $(container) zsh
@@ -26,6 +31,6 @@ remove-dev-container:
test: remove-dev-container
docker run -p 5431:5432 \
--name $(container)-dev \
$$(cat .env | sed 's/^/-e /') \
--env-file .env \
-v ./pgsql-data-dev/:/var/lib/postgresql/data/ \
-it $(image)
+4
View File
@@ -0,0 +1,4 @@
#!/bin/bash
set -e
pg_isready -U "$ExpDB_USER" -d "$ExpDB_NAME"