Added a script to migrate the dumped CSV data from the sqlite3 DB to PG
This commit is contained in:
@@ -0,0 +1,126 @@
|
||||
#!/bin/env python
|
||||
|
||||
import psycopg2
|
||||
from psycopg2 import Error
|
||||
|
||||
|
||||
class DB():
|
||||
def __init__(self, user, passwd,
|
||||
db, host, port):
|
||||
try:
|
||||
self.conn = psycopg2.connect(user=user, password=passwd, database=db,
|
||||
host=host, port=port)
|
||||
self.cursor = self.conn.cursor()
|
||||
except (Exception, Error) as error:
|
||||
raise error
|
||||
|
||||
|
||||
def runQuery(self, query, data):
|
||||
try:
|
||||
print(f"{query} {data}")
|
||||
self.cursor.execute(query, data)
|
||||
self.conn.commit()
|
||||
except (Exception, Error) as error:
|
||||
raise error
|
||||
|
||||
|
||||
def get_data(filePath):
|
||||
rows = []
|
||||
with open(filePath, "r") as f:
|
||||
for line in f:
|
||||
fields = [v.strip('"') for v in line.strip().split(",")]
|
||||
rows.append(fields)
|
||||
|
||||
return rows
|
||||
|
||||
|
||||
def truncate_table(db, m):
|
||||
print("Truncating table:", m["table"])
|
||||
try:
|
||||
db.runQuery(f"TRUNCATE TABLE {m["table"]} CASCADE", [])
|
||||
except (Exception, Error) as error:
|
||||
raise error
|
||||
|
||||
|
||||
def insert_data(db, query, data):
|
||||
for row in data:
|
||||
try:
|
||||
db.runQuery(query, row)
|
||||
except (Exception, Error) as error:
|
||||
raise error
|
||||
|
||||
|
||||
def mig_data(db, m):
|
||||
print("Running migration for table:", m["table"])
|
||||
|
||||
values = ','.join(['%s'] * len(m["headers"]))
|
||||
fields = ','.join(m["headers"])
|
||||
query = f"INSERT INTO {m["table"]} ({fields}) VALUES({values})"
|
||||
|
||||
try:
|
||||
data = get_data(m["csvFile"])
|
||||
insert_data(db, query, data)
|
||||
except (Exception, Error) as error:
|
||||
raise error
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
dbObj = DB("expuser", "exppass", "expdb", "127.0.0.1", "5431")
|
||||
except (Exception, Error) as error:
|
||||
print("Threw error:", error)
|
||||
exit(0)
|
||||
|
||||
migs = [
|
||||
{
|
||||
"table": "schema_migrations",
|
||||
"csvFile": "./schema_migrations.csv",
|
||||
"headers": ["version", "dirty"],
|
||||
},
|
||||
{
|
||||
"table": "categories",
|
||||
"csvFile": "./categories.csv",
|
||||
"headers": ["CategoryID", "CategoryName"],
|
||||
},
|
||||
{
|
||||
"table": "users",
|
||||
"csvFile": "./users.csv",
|
||||
"headers": ["UserID", "UserName", "UserPass"],
|
||||
},
|
||||
{
|
||||
"table": "stores",
|
||||
"csvFile": "./stores.csv",
|
||||
"headers": ["StoreID", "StoreName"],
|
||||
},
|
||||
{
|
||||
"table": "expenseTypes",
|
||||
"csvFile": "./expenseTypes.csv",
|
||||
"headers": ["TypeID", "TypeName"],
|
||||
},
|
||||
{
|
||||
"table": "expenses",
|
||||
"csvFile": "./expenses.csv",
|
||||
"headers": ["ExpID", "Description", "Value", "StoreID",
|
||||
"CategoryID", "OwnerUserID", "TypeID", "ExpDate",
|
||||
"CreationDate", "PaidOff", "SharesEven"],
|
||||
},
|
||||
{
|
||||
"table": "expensesShares",
|
||||
"csvFile": "./expensesShares.csv",
|
||||
"headers": ["ExpShareID", "ExpID", "UserID", "Share", "Calculated"],
|
||||
},
|
||||
{
|
||||
"table": "expensesPayments",
|
||||
"csvFile": "./expensesPayments.csv",
|
||||
"headers": ["ExpPaymID", "ExpID", "UserID", "Payed"],
|
||||
},
|
||||
]
|
||||
|
||||
for m in migs:
|
||||
truncate_table(dbObj, m)
|
||||
|
||||
for m in migs:
|
||||
try:
|
||||
mig_data(dbObj, m)
|
||||
except (Exception, Error) as error:
|
||||
print("failed to migrate data:", error)
|
||||
@@ -0,0 +1,2 @@
|
||||
schema.sql
|
||||
.env
|
||||
Reference in New Issue
Block a user