Merge branch 'yifan' of ASJ_ADS/sync_amz_data into master

start_sync_amz.py

@@ -4,67 +4,89 @@ from sync_amz_data.DataTransform import Data_ETL
 from sync_amz_data.DataTransform.Data_ETL import Common_ETLMethod,SP_ETL,SB_ETL,SD_ETL
 import logging.config
 from apscheduler.schedulers.blocking import BlockingScheduler
+logging.config.dictConfig(LOG_CONF)
 
+def amz_report(AWS_CREDENTIALS,para={}):
+    conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+    sb_report = SB_ETL(**AWS_CREDENTIALS)
+    sb_report.reportV3_purchasedAsinRecord_ETL(conn,params=para)
+    sb_report.reportV2_campaignsRecord_ETL(conn,params=para)
+    sb_report.reportV2_campaignsVideo_ETL(conn,params=para)
+    sb_report.reportV2_adGroupsRecord_ETL(conn,params=para)
+    sb_report.reportV2_adGroupsVideo_ETL(conn,params=para)
+    sb_report.reportV2_adsRecord_ETL(conn,params=para)
+    sb_report.reportV2_adsVideo_ETL(conn,params=para)
+    sb_report.reportV2_keywordsRecord_ETL(conn,params=para)
+    sb_report.reportV2_keywordsVideo_ETL(conn,params=para)
+    sb_report.reportV2_placementRecord_ETL(conn,params=para)
+    sb_report.reportV2_placementVideo_ETL(conn,params=para)
+    sb_report.reportV2_searchtermsRecord_ETL(conn,params=para)
+    sb_report.reportV2_searchtermsVideo_ETL(conn,params=para)
+    sb_report.reportV2_targetsRecord_ETL(conn,params=para)
+    sb_report.reportV2_targetsVideo_ETL(conn,params=para)
+    conn.close()
+
+    conn = SP_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+    sp_report = SP_ETL(**AWS_CREDENTIALS)
+    sp_report.reportV3_campaign_spCampaignsETL(conn,params=para)
+    sp_report.reportV3_adGroup_spCampaignsETL(conn,params=para)
+    sp_report.reportV3_campaignPlacement_spCampaignsETL(conn,params=para)
+    sp_report.reportV3_targeting_spTargetingETL(conn,params=para)
+    sp_report.reportV3_searchTerm_spSearchTermETL(conn,params=para)
+    sp_report.reportV3_advertiser_spAdvertisedProductETL(conn,params=para)
+    sp_report.reportV3_asin_spPurchasedProductETL(conn,params=para)
+    conn.close()
+
+    conn = SD_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+    sd_report = SD_ETL(**AWS_CREDENTIALS)
+
+    sd_report.reportV3_campaign_sdCampaigns_ETL(conn,params=para)
+    sd_report.reportV3_campaignMT_sdCampaigns_ETL(conn,params=para)
+    sd_report.reportV3_adgroup_sdAdGroup_ETL(conn,params=para)
+    sd_report.reportV3_adgroupMT_sdAdGroup_ETL(conn,params=para)
+    sd_report.reportV3_targeting_sdTargeting_ETL(conn,params=para)
+    sd_report.reportV3_targetingMT_sdTargeting_ETL(conn,params=para)
+    sd_report.reportV3_asin_sdPurchasedProduct_ETL(conn,params=para)
+    sd_report.reportV3_advertiser_sdAdvertisedProduct_ETL(conn,params=para)
+
+    # sd_report.reportV2_campaignsRecord_t2_ETL(conn,params=para)
+    # sd_report.reportV2_campaignsRecord_t3_ETL(conn,params=para)
+    # sd_report.reportV2_adGroupsRecord_t2_ETL(conn,params=para)
+    # sd_report.reportV2_adGroupsRecord_t3_ETL(conn,params=para)
+    # sd_report.reportV2_asins_t2_ETL(conn,params=para)
+    # sd_report.reportV2_asins_t3_ETL(conn,params=para)
+    # sd_report.reportV2_productAds_t2_ETL(conn,params=para)
+    # sd_report.reportV2_productAds_t3_ETL(conn,params=para)
+    # sd_report.reportV2_targets_t2_ETL(conn,params=para)
+    # sd_report.reportV2_targets_t3_ETL(conn,params=para)
+    # sd_report.reportV2_campaign_matchedTarget_t2_ETL(conn,params=para)
+    # sd_report.reportV2_campaign_matchedTarget_t3_ETL(conn,params=para)
+    # sd_report.reportV2_adGroups_matchedTarget_t2_ETL(conn,params=para)
+    # sd_report.reportV2_adGroups_matchedTarget_t3_ETL(conn,params=para)
+    # sd_report.reportV2_targets_matchedTarget_t2_ETL(conn,params=para)
+    # sd_report.reportV2_targets_matchedTarget_t3_ETL(conn,params=para)
+    conn.close()
 
 
-logging.config.dictConfig(LOG_CONF)
 
 if __name__ == '__main__':
     # AccountTask("3006125408623189").do({"record": "portfolios"})
-
     AWS_CREDENTIALS = {
         'lwa_client_id': 'amzn1.application-oa2-client.ebd701cd07854fb38c37ee49ec4ba109',
         'refresh_token': "Atzr|IwEBIL4ur8kbcwRyxVu_srprAAoTYzujnBvA6jU-0SMxkRgOhGjYJSUNGKvw24EQwJa1jG5RM76mQD2P22AKSq8qSD94LddoXGdKDO74eQVYl0RhuqOMFqdrEZpp1p4bIR6_N8VeSJDHr7UCuo8FiabkSHrkq7tsNvRP-yI-bnpQv4EayPBh7YwHVX3hYdRbhxaBvgJENgCuiEPb35Q2-Z6w6ujjiKUAK2VSbCFpENlEfcHNsjDeY7RCvFlwlCoHj1IeiNIaFTE9yXFu3aEWlExe3LzHv6PZyunEi88QJSXKSh56Um0e0eEg05rMv-VBM83cAqc5POmZnTP1vUdZO8fQv3NFLZ-xU6e1WQVxVPi5Cyqk4jYhGf1Y9t98N654y0tVvw74qNIsTrB-8bGS0Uhfe24oBEWmzObvBY3zhtT1d42myGUJv4pMTU6yPoS83zhPKm3LbUDEpBA1hvvc_09jHk7vUEAuFB-UAZzlht2C1yklzQ",
         'lwa_client_secret': 'cbf0514186db4df91e04a8905f0a91b605eae4201254ced879d8bb90df4b474d',
         'profile_id': "3006125408623189"
     }
-    conn = Common_ETLMethod(**AWS_CREDENTIALS).clickhouse_connect()
-    timezone = Common_ETLMethod(**AWS_CREDENTIALS).timeZone()
-    print(timezone)
-
+    timezone_ = Common_ETLMethod(**AWS_CREDENTIALS).timeZone()
+    print(timezone_)
     sched = BlockingScheduler()
-    sb_report = SB_ETL(**AWS_CREDENTIALS)
-    sched.add_job(lambda: sb_report.reportV3_purchasedAsinRecord_ETL(conn),'cron',hour=6,minute=45,second=0,timezone=timezone)
-    sched.add_job(lambda: sb_report.reportV2_campaignsRecord_ETL(conn),'cron',hour=6,minute=50,second=0,timezone=timezone)
-    sched.add_job(lambda: sb_report.reportV2_campaignsVideo_ETL(conn),'cron',hour=6,minute=55,second=0,timezone=timezone)
-    sched.add_job(lambda: sb_report.reportV2_adGroupsRecord_ETL(conn),'cron',hour=7,minute=0,second=0,timezone=timezone)
-    sched.add_job(lambda: sb_report.reportV2_adGroupsVideo_ETL(conn),'cron',hour=7,minute=5,second=0,timezone=timezone)
-    sched.add_job(lambda: sb_report.reportV2_adsRecord_ETL(conn),'cron',hour=7,minute=10,second=0,timezone=timezone)
-    sched.add_job(lambda: sb_report.reportV2_adsVideo_ETL(conn),'cron',hour=7,minute=15,second=0,timezone=timezone)
-    sched.add_job(lambda: sb_report.reportV2_keywordsRecord_ETL(conn),'cron',hour=7,minute=20,second=0,timezone=timezone)
-    sched.add_job(lambda: sb_report.reportV2_keywordsVideo_ETL(conn),'cron',hour=7,minute=25,second=0,timezone=timezone)
-    sched.add_job(lambda: sb_report.reportV2_placementRecord_ETL(conn),'cron',hour=7,minute=30,second=0,timezone=timezone)
-    sched.add_job(lambda: sb_report.reportV2_placementVideo_ETL(conn),'cron',hour=7,minute=35,second=0,timezone=timezone)
-    sched.add_job(lambda: sb_report.reportV2_searchtermsRecord_ETL(conn),'cron',hour=7,minute=40,second=0,timezone=timezone)
-    sched.add_job(lambda: sb_report.reportV2_searchtermsVideo_ETL(conn),'cron',hour=7,minute=45,second=0,timezone=timezone)
-    sched.add_job(lambda: sb_report.reportV2_targetsRecord_ETL(conn),'cron',hour=7,minute=50,second=0,timezone=timezone)
-    sched.add_job(lambda: sb_report.reportV2_targetsVideo_ETL(conn),'cron',hour=7,minute=55,second=0,timezone=timezone)
+    sched.add_job(lambda: amz_report(AWS_CREDENTIALS=AWS_CREDENTIALS),'cron',hour=7,minute=0,second=0,timezone=timezone_)#,params={"startDate":"2023-11-04","endDate":"2023-11-04","date":"20231104"}
 
-    sp_report = SP_ETL(**AWS_CREDENTIALS)
-    sched.add_job(lambda: sp_report.reportV3_campaign_spCampaignsETL(conn),'cron',hour=8,minute=0,second=0,timezone=timezone)
-    sched.add_job(lambda: sp_report.reportV3_adGroup_spCampaignsETL(conn),'cron',hour=8,minute=5,second=0,timezone=timezone)
-    sched.add_job(lambda: sp_report.reportV3_campaignPlacement_spCampaignsETL(conn),'cron',hour=8,minute=10,second=0,timezone=timezone)
-    sched.add_job(lambda: sp_report.reportV3_targeting_spTargetingETL(conn),'cron',hour=8,minute=15,second=0,timezone=timezone)
-    sched.add_job(lambda: sp_report.reportV3_searchTerm_spSearchTermETL(conn),'cron',hour=8,minute=20,second=0,timezone=timezone)
-    sched.add_job(lambda: sp_report.reportV3_advertiser_spAdvertisedProductETL(conn),'cron',hour=8,minute=25,second=0,timezone=timezone)
-    sched.add_job(lambda: sp_report.reportV3_asin_spPurchasedProductETL(conn),'cron',hour=8,minute=30,second=0,timezone=timezone)
+    sched.start()
 
-    sd_report = SD_ETL(**AWS_CREDENTIALS)
-    sched.add_job(lambda: sd_report.reportV2_campaignsRecord_t2_ETL(conn),'cron',hour=8,minute=35,second=0,timezone=timezone)
-    sched.add_job(lambda: sd_report.reportV2_campaignsRecord_t3_ETL(conn),'cron',hour=8,minute=40,second=0,timezone=timezone)
-    sched.add_job(lambda: sd_report.reportV2_adGroupsRecord_t2_ETL(conn),'cron',hour=8,minute=45,second=0,timezone=timezone)
-    sched.add_job(lambda: sd_report.reportV2_adGroupsRecord_t3_ETL(conn),'cron',hour=8,minute=50,second=0,timezone=timezone)
-    sched.add_job(lambda: sd_report.reportV2_asins_t2_ETL(conn),'cron',hour=8,minute=55,second=0,timezone=timezone)
-    sched.add_job(lambda: sd_report.reportV2_asins_t3_ETL(conn),'cron',hour=9,minute=0,second=0,timezone=timezone)
-    sched.add_job(lambda: sd_report.reportV2_productAds_t2_ETL(conn),'cron',hour=9,minute=5,second=0,timezone=timezone)
-    sched.add_job(lambda: sd_report.reportV2_productAds_t3_ETL(conn),'cron',hour=9,minute=10,second=0,timezone=timezone)
-    sched.add_job(lambda: sd_report.reportV2_targets_t2_ETL(conn),'cron',hour=9,minute=15,second=0,timezone=timezone)
-    sched.add_job(lambda: sd_report.reportV2_productAds_t3_ETL(conn),'cron',hour=9,minute=20,second=0,timezone=timezone)
-    sched.add_job(lambda: sd_report.reportV2_campaign_matchedTarget_t2_ETL(conn),'cron',hour=9,minute=25,second=0,timezone=timezone)
-    sched.add_job(lambda: sd_report.reportV2_campaign_matchedTarget_t3_ETL(conn),'cron',hour=9,minute=30,second=0,timezone=timezone)
-    sched.add_job(lambda: sd_report.reportV2_adGroups_matchedTarget_t2_ETL(conn),'cron',hour=9,minute=35,second=0,timezone=timezone)
-    sched.add_job(lambda: sd_report.reportV2_adGroups_matchedTarget_t3_ETL(conn),'cron',hour=9,minute=40,second=0,timezone=timezone)
-    sched.add_job(lambda: sd_report.reportV2_targets_matchedTarget_t2_ETL(conn),'cron',hour=9,minute=45,second=0,timezone=timezone)
-    sched.add_job(lambda: sd_report.reportV2_targets_matchedTarget_t3_ETL(conn),'cron',hour=9,minute=50,second=0,timezone=timezone)
 
-    sched.start()
+    # list_date = ["2023-11-05",]
+    # for date_ in list_date:
+    #     print(date_)
+    #     print(date_.replace("-",""))
+    #     amz_report(AWS_CREDENTIALS,para={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})