Can we fully use dap initdb and syncdb entirely and does this supports big data initiallizing?

GabrielBabierra
Community Member

Can we fully use dap initdb and syncdb to our production usage? When i was testing this command line, it doesnt seem to work for a table that contains a huge data, total of 4 sliced objects for one table which is submissions. 

 

Error message:

2023-10-26 15:56:24,721 - ERROR - Response payload is not completed

Traceback (most recent call last):
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/dap/__main__.py", line 133, in console_entry
main()
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/dap/__main__.py", line 125, in main
asyncio.run(dapCommand.execute(args))
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/asyncio/runners.py", line 44, in run
return loop.run_until_complete(main)
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/asyncio/base_events.py", line 616, in run_until_complete
return future.result()
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/dap/commands/commands.py", line 31, in execute
executed = await super().execute(args)
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/dap/commands/base.py", line 49, in execute
if await subcommand.execute(args):
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/dap/commands/base.py", line 45, in execute
await self._execute_impl(args)
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/dap/commands/initdb_command.py", line 31, in _execute_impl
await init_db(
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/dap/actions/init_db.py", line 16, in init_db
await SQLReplicator(session, db_connection).initialize(
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/dap/replicator/sql.py", line 67, in initialize
await client.download(
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/dap/downloader.py", line 83, in download
await wait_n(
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/dap/concurrency.py", line 49, in wait_n
raise exc
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/dap/downloader.py", line 81, in logged_download_and_save
await self._download(db_lock, context_aware_object, processor=processor)
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/dap/downloader.py", line 113, in _download
await processor.process(obj, records)
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/dap/integration/base_processor.py", line 19, in process
await self.process_impl(obj, self._convert(records))
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/dap/plugins/mysql/init_processor.py", line 71, in process_impl
await self._db_connection.execute(
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/dap/integration/base_connection.py", line 41, in execute
return await query(self._raw_connection)
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/dap/plugins/mysql/queries.py", line 26, in __call__
return await self._query_func(mysql_conn)
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/dap/plugins/mysql/init_processor.py", line 83, in _execute_with_connection
records_to_insert = await self._convert_records(records)
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/dap/plugins/mysql/init_processor.py", line 90, in _convert_records
async for record in records:
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/dap/integration/base_processor.py", line 30, in _convert
async for record in records:
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/dap/payload.py", line 21, in get_lines_from_gzip_stream
compressed_data = await stream.read(64 * 1024)
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/aiohttp/streams.py", line 385, in read
await self._wait("read")
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/aiohttp/streams.py", line 304, in _wait
await waiter
aiohttp.client_exceptions.ClientPayloadError: Response payload is not completed

0 Likes
Users who also had this question