Explorar o código

modidy Data_ETL

huangyifan hai 1 ano
pai
achega
8b28cccf5e
Modificáronse 3 ficheiros con 254 adicións e 300 borrados
  1. 6 6
      start_sync_amz.py
  2. 7 7
      start_sync_amz_RightNowRun.py
  3. 241 287
      sync_amz_data/DataTransform/Data_ETL.py

+ 6 - 6
start_sync_amz.py

@@ -1,13 +1,13 @@
-from sync_amz_data.tasks.account import AccountTask
-from sync_amz_data.settings import LOG_CONF
-import logging.config
-logging.config.dictConfig(LOG_CONF)
+# from sync_amz_data.tasks.account import AccountTask
+# from sync_amz_data.settings import LOG_CONF
+# import logging.config
+# logging.config.dictConfig(LOG_CONF)
 from sync_amz_data.DataTransform import Data_ETL
 from sync_amz_data.DataTransform.Data_ETL import Common_ETLMethod,SP_ETL,SB_ETL,SD_ETL
 from apscheduler.schedulers.blocking import BlockingScheduler
 
 
-def amz_report(AWS_CREDENTIALS,para={}):
+def amz_report(AWS_CREDENTIALS,para=None):
     conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
     sb_report = SB_ETL(**AWS_CREDENTIALS)
     sb_report.reportV3_purchasedAsinRecord_ETL(conn,params=para)
@@ -80,7 +80,7 @@ if __name__ == '__main__':
     timezone_ = Common_ETLMethod(**AWS_CREDENTIALS).timeZone()
     print(timezone_)
     sched = BlockingScheduler()
-    sched.add_job(amz_report,'cron',hour=17,minute=0,second=0,timezone=timezone_, args=(AWS_CREDENTIALS, {}))#,params={"startDate":"2023-11-04","endDate":"2023-11-04","date":"20231104"}
+    sched.add_job(amz_report,'cron',hour=17,minute=0,second=0,timezone=timezone_, args=(AWS_CREDENTIALS, ))#,params={"startDate":"2023-11-04","endDate":"2023-11-04","date":"20231104"}
     sched.start()
 
 

+ 7 - 7
start_sync_amz_RightNowRun.py

@@ -1,12 +1,12 @@
-from sync_amz_data.tasks.account import AccountTask
-from sync_amz_data.settings import LOG_CONF
-import logging.config
-logging.config.dictConfig(LOG_CONF)
+# from sync_amz_data.tasks.account import AccountTask
+# from sync_amz_data.settings import LOG_CONF
+# import logging.config
+# logging.config.dictConfig(LOG_CONF)
 from sync_amz_data.DataTransform import Data_ETL
 from sync_amz_data.DataTransform.Data_ETL import Common_ETLMethod,SP_ETL,SB_ETL,SD_ETL
 
 
-def amz_report(AWS_CREDENTIALS,para={}):
+def amz_report(AWS_CREDENTIALS,para=None):
     conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
     sb_report = SB_ETL(**AWS_CREDENTIALS)
     sb_report.reportV3_purchasedAsinRecord_ETL(conn,params=para)
@@ -81,8 +81,8 @@ if __name__ == '__main__':
     amz_report(AWS_CREDENTIALS=AWS_CREDENTIALS)
 
 
-    # list_date = ['2023-10-23']
-    # # list_date = [f'2023-10-{i}' for i in range(24,23,-1)]
+    # list_date = ['2023-11-12','2023-11-13']
+    # # # list_date = [f'2023-10-{i}' for i in range(24,23,-1)]
     # for date_ in list_date:
     #     print(date_)
     #     print(date_.replace("-",""))

A diferenza do arquivo foi suprimida porque é demasiado grande
+ 241 - 287
sync_amz_data/DataTransform/Data_ETL.py


Algúns arquivos non se mostraron porque demasiados arquivos cambiaron neste cambio