From 27766d8c0f4076080051b6372713946a958bc235 Mon Sep 17 00:00:00 2001 From: Kristofer Spinka Date: Sat, 17 Apr 2021 12:56:36 -0400 Subject: [PATCH] SQLAlchemy must be flushed to avoid running out of memory If we don't call flush, all of the changes will accumulate in memory and eventually exhaust the host's memory where socrata2sql is running. This does not mean that the transaction is committed, it just means that the rows are being written to the database and not queued in the tool. --- socrata2sql/cli.py | 1 + 1 file changed, 1 insertion(+) diff --git a/socrata2sql/cli.py b/socrata2sql/cli.py index 0497822..75c1375 100644 --- a/socrata2sql/cli.py +++ b/socrata2sql/cli.py @@ -310,6 +310,7 @@ def main(): to_insert.append(Binding(**parse_row(row, Binding))) session.add_all(to_insert) + session.flush() bar.next(n=len(to_insert)) bar.finish()