Merge branch 'yifan' of ASJ_ADS/sync_amz_data into master

start_sync_amz.py

@@ -1,5 +1,5 @@
 import time
-
+import pandas as pd
 from sync_amz_data.settings import LOG_CONF
 import logging.config
 logging.config.dictConfig(LOG_CONF)
@@ -8,125 +8,246 @@ from sync_amz_data.DataTransform.Data_ETL import Common_ETLMethod,SP_ETL,SB_ETL,
 from apscheduler.schedulers.blocking import BlockingScheduler
 from datetime import datetime,timedelta
 
-def amz_report(AWS_CREDENTIALS,para=None):
-    count = 1
-    list_date = [((datetime.today()+timedelta(days=i)).date()).isoformat() for i in range(-2,-10,-1)]
-    conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
-    # conn.command('')
-    if para is not None:
-        list_date = [para['startDate']]
-    # print(list_date)
-    for date_ in list_date:
-        print(date_)
-        print(date_.replace("-", ""))
-        # amz_report(conn, AWS_CREDENTIALS, para={"startDate": date_, "endDate": date_, "date": date_.replace("-", "")})
-        if count==1:
-            delete_list = ['SB_adGroupsV2',
-                            'SB_adGroupsVideoV2',
-                            'SB_adsV2',
-                            'SB_adsVideoV2',
-                            'SB_campaignsPlacementV2',
-                            'SB_campaignsPlacementVideoV2',
-                            'SB_campaignsV2',
-                            'SB_campaignsVideoV2',
-                            'SB_keywordsQueryV2',
-                            'SB_keywordsQueryVideoV2',
-                            'SB_keywordsV2',
-                            'SB_keywordsVideoV2',
-                            'SB_sbAdGroup_adGroupV3',
-                            'SB_sbAds_adsV3',
-                            'SB_sbCampaigns_campaignV3',
-                            'SB_sbCampaigns_placementV3',
-                            'SB_sbPurchasedProduct_asinV3',
-                            'SB_sbSearchTerm_searchTermV3',
-                            'SB_sbTargeting_targetingV3',
-                            'SB_targetsV2',
-                            'SB_targetsVideoV2',
-                            'SD_adGroupsMatchedTargetV2',
-                            'SD_adGroupsV2',
-                            'SD_adsV2',
-                            'SD_advertiser_sdAdvertisedProductV3',
-                            'SD_asin_sdPurchasedProductV3',
-                            'SD_asinsV2',
-                            'SD_campaignsMatchedTargetV2',
-                            'SD_campaignsV2',
-                            'SD_sdAdGroup_adGroupMatchedTargetV3',
-                            'SD_sdAdGroup_adGroupV3',
-                            'SD_sdCampaigns_campaignMatchedTargetV3',
-                            'SD_sdCampaigns_campaignV3',
-                            'SD_targeting_sdTargetingMatchedTargetV3',
-                            'SD_targeting_sdTargetingV3',
-                            'SD_targetsMatchedTargetV2',
-                            'SD_targetsV2',
-                            'SP_spAdvertisedProduct_advertiserV3',
-                            'SP_spCampaigns_adGroupV3',
-                            'SP_spCampaigns_campaignV3',
-                            'SP_spCampaigns_placementV3',
-                            'SP_spPurchasedProduct_asinV3',
-                            'SP_spSearchTerm_searchTermV3',
-                            'SP_spTargeting_targetingV3']
-            for i in delete_list:
-                conn.command(f"delete from AmazonReport.{i} where date='{list_date[-2]}'")
-            count+=1
+def pre_func(AWS_CREDENTIALS,date_):
 
-        try:
-            refresh_token = shop_infos(AWS_CREDENTIALS['profile_id'])['refresh_token']
-            AWS_CREDENTIALS['refresh_token'] = refresh_token
-        except Exception as e:
-            print(e)
 
         conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
         sb_report = SB_ETL(**AWS_CREDENTIALS)
-        sb_report.reportV3_campaign_sbCampaigns_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        sb_report.reportV3_adGroup_sbAdGroup_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        sb_report.reportV3_sbCampaignPlacement_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        sb_report.reportV3_sbTargeting_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        sb_report.reportV3_sbSearchTerm_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        sb_report.reportV3_sbAds_ETL(conn, params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        sb_report.reportV3_purchasedAsinRecord_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-
-        sb_report.reportV3_purchasedAsinRecord_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        sb_report.reportV2_campaignsRecord_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        sb_report.reportV2_campaignsVideo_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        sb_report.reportV2_adGroupsRecord_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        sb_report.reportV2_adGroupsVideo_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        sb_report.reportV2_adsRecord_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        sb_report.reportV2_adsVideo_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        sb_report.reportV2_keywordsRecord_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        sb_report.reportV2_keywordsVideo_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        sb_report.reportV2_placementRecord_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        sb_report.reportV2_placementVideo_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        sb_report.reportV2_searchtermsRecord_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        sb_report.reportV2_searchtermsVideo_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        sb_report.reportV2_targetsRecord_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        sb_report.reportV2_targetsVideo_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
+        try:
+            sb_report.reportV3_campaign_sbCampaigns_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
+        except:
+            conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+            sb_report.reportV3_campaign_sbCampaigns_ETL(conn, params={"startDate": date_, "endDate": date_,
+                                                                      "date": date_.replace("-", "")})
+        try:
+            sb_report.reportV3_adGroup_sbAdGroup_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
+        except:
+            conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+            sb_report.reportV3_adGroup_sbAdGroup_ETL(conn, params={"startDate": date_, "endDate": date_,
+                                                                      "date": date_.replace("-", "")})
+        try:
+            sb_report.reportV3_sbCampaignPlacement_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
+        except:
+            conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+            sb_report.reportV3_sbCampaignPlacement_ETL(conn, params={"startDate": date_, "endDate": date_,
+                                                                      "date": date_.replace("-", "")})
+        try:
+            sb_report.reportV3_sbTargeting_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
+        except:
+            conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+            sb_report.reportV3_sbTargeting_ETL(conn, params={"startDate": date_, "endDate": date_,
+                                                                      "date": date_.replace("-", "")})
+        try:
+            sb_report.reportV3_sbSearchTerm_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
+        except:
+            conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+            sb_report.reportV3_sbSearchTerm_ETL(conn, params={"startDate": date_, "endDate": date_,
+                                                                      "date": date_.replace("-", "")})
+        try:
+            sb_report.reportV3_sbAds_ETL(conn, params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
+        except:
+            conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+            sb_report.reportV3_sbAds_ETL(conn, params={"startDate": date_, "endDate": date_,
+                                                                      "date": date_.replace("-", "")})
+        try:
+            sb_report.reportV3_purchasedAsinRecord_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
+        except:
+            conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+            sb_report.reportV3_purchasedAsinRecord_ETL(conn, params={"startDate": date_, "endDate": date_,
+                                                                      "date": date_.replace("-", "")})
+        try:
+            sb_report.reportV3_purchasedAsinRecord_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
+        except:
+            conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+            sb_report.reportV3_purchasedAsinRecord_ETL(conn, params={"startDate": date_, "endDate": date_,
+                                                                      "date": date_.replace("-", "")})
+        try:
+            sb_report.reportV2_campaignsRecord_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
+        except:
+            conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+            sb_report.reportV2_campaignsRecord_ETL(conn, params={"startDate": date_, "endDate": date_,
+                                                                      "date": date_.replace("-", "")})
+        try:
+            sb_report.reportV2_campaignsVideo_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
+        except:
+            conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+            sb_report.reportV2_campaignsVideo_ETL(conn, params={"startDate": date_, "endDate": date_,
+                                                                      "date": date_.replace("-", "")})
+        try:
+            sb_report.reportV2_adGroupsRecord_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
+        except:
+            conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+            sb_report.reportV2_adGroupsRecord_ETL(conn, params={"startDate": date_, "endDate": date_,
+                                                                      "date": date_.replace("-", "")})
+
+        try:
+            sb_report.reportV2_adGroupsVideo_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
+        except:
+            conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+            sb_report.reportV2_adGroupsVideo_ETL(conn, params={"startDate": date_, "endDate": date_,
+                                                                      "date": date_.replace("-", "")})
+        try:
+            sb_report.reportV2_adsRecord_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
+        except:
+            conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+            sb_report.reportV2_adsRecord_ETL(conn, params={"startDate": date_, "endDate": date_,
+                                                                      "date": date_.replace("-", "")})
+        try:
+            sb_report.reportV2_adsVideo_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
+        except:
+            conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+            sb_report.reportV2_adsVideo_ETL(conn, params={"startDate": date_, "endDate": date_,
+                                                                      "date": date_.replace("-", "")})
+        try:
+            sb_report.reportV2_keywordsRecord_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
+        except:
+            conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+            sb_report.reportV2_keywordsRecord_ETL(conn, params={"startDate": date_, "endDate": date_,
+                                                                      "date": date_.replace("-", "")})
+        try:
+            sb_report.reportV2_keywordsVideo_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
+        except:
+            conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+            sb_report.reportV2_keywordsVideo_ETL(conn, params={"startDate": date_, "endDate": date_,
+                                                                      "date": date_.replace("-", "")})
+        try:
+            sb_report.reportV2_placementRecord_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
+        except:
+            conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+            sb_report.reportV2_placementRecord_ETL(conn, params={"startDate": date_, "endDate": date_,
+                                                                      "date": date_.replace("-", "")})
+        try:
+            sb_report.reportV2_placementVideo_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
+        except:
+            conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+            sb_report.reportV2_placementVideo_ETL(conn, params={"startDate": date_, "endDate": date_,
+                                                                      "date": date_.replace("-", "")})
+        try:
+            sb_report.reportV2_searchtermsRecord_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
+        except:
+            conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+            sb_report.reportV2_searchtermsRecord_ETL(conn, params={"startDate": date_, "endDate": date_,
+                                                                      "date": date_.replace("-", "")})
+        try:
+            sb_report.reportV2_searchtermsVideo_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
+        except:
+            conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+            sb_report.reportV2_searchtermsVideo_ETL(conn, params={"startDate": date_, "endDate": date_,
+                                                                      "date": date_.replace("-", "")})
+        try:
+            sb_report.reportV2_targetsRecord_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
+        except:
+            conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+            sb_report.reportV2_targetsRecord_ETL(conn, params={"startDate": date_, "endDate": date_,
+                                                                      "date": date_.replace("-", "")})
+        try:
+            sb_report.reportV2_targetsVideo_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
+        except:
+            conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+            sb_report.reportV2_targetsVideo_ETL(conn, params={"startDate": date_, "endDate": date_,
+                                                                      "date": date_.replace("-", "")})
+
         conn.close()
         time.sleep(5)
 
         conn = SP_ETL(**AWS_CREDENTIALS).clickhouse_connect()
         sp_report = SP_ETL(**AWS_CREDENTIALS)
-        sp_report.reportV3_campaign_spCampaignsETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        sp_report.reportV3_adGroup_spCampaignsETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        sp_report.reportV3_campaignPlacement_spCampaignsETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        sp_report.reportV3_targeting_spTargetingETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        sp_report.reportV3_searchTerm_spSearchTermETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        sp_report.reportV3_advertiser_spAdvertisedProductETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        sp_report.reportV3_asin_spPurchasedProductETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
+        try:
+            sp_report.reportV3_campaign_spCampaignsETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
+        except:
+            conn = SP_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+            sp_report.reportV3_campaign_spCampaignsETL(conn, params={"startDate": date_, "endDate": date_,
+                                                                      "date": date_.replace("-", "")})
+        try:
+            sp_report.reportV3_adGroup_spCampaignsETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
+        except:
+            conn = SP_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+            sp_report.reportV3_adGroup_spCampaignsETL(conn, params={"startDate": date_, "endDate": date_,
+                                                                      "date": date_.replace("-", "")})
+        try:
+            sp_report.reportV3_campaignPlacement_spCampaignsETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
+        except:
+            conn = SP_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+            sp_report.reportV3_campaignPlacement_spCampaignsETL(conn, params={"startDate": date_, "endDate": date_,
+                                                                      "date": date_.replace("-", "")})
+        try:
+            sp_report.reportV3_targeting_spTargetingETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
+        except:
+            conn = SP_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+            sp_report.reportV3_targeting_spTargetingETL(conn, params={"startDate": date_, "endDate": date_,
+                                                                      "date": date_.replace("-", "")})
+        try:
+            sp_report.reportV3_searchTerm_spSearchTermETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
+        except:
+            conn = SP_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+            sp_report.reportV3_searchTerm_spSearchTermETL(conn, params={"startDate": date_, "endDate": date_,
+                                                                      "date": date_.replace("-", "")})
+        try:
+            sp_report.reportV3_advertiser_spAdvertisedProductETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
+        except:
+            conn = SP_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+            sp_report.reportV3_advertiser_spAdvertisedProductETL(conn, params={"startDate": date_, "endDate": date_,
+                                                                      "date": date_.replace("-", "")})
+        try:
+            sp_report.reportV3_asin_spPurchasedProductETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
+        except:
+            conn = SP_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+            sp_report.reportV3_asin_spPurchasedProductETL(conn, params={"startDate": date_, "endDate": date_,
+                                                                      "date": date_.replace("-", "")})
         conn.close()
         time.sleep(5)
 
 
         conn = SD_ETL(**AWS_CREDENTIALS).clickhouse_connect()
         sd_report = SD_ETL(**AWS_CREDENTIALS)
-
-        sd_report.reportV3_campaign_sdCampaigns_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        sd_report.reportV3_campaignMT_sdCampaigns_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        sd_report.reportV3_adgroup_sdAdGroup_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        sd_report.reportV3_adgroupMT_sdAdGroup_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        sd_report.reportV3_targeting_sdTargeting_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        sd_report.reportV3_targetingMT_sdTargeting_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        sd_report.reportV3_asin_sdPurchasedProduct_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        sd_report.reportV3_advertiser_sdAdvertisedProduct_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
+        try:
+            sd_report.reportV3_campaign_sdCampaigns_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
+        except:
+            conn = SD_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+            sd_report.reportV3_campaign_sdCampaigns_ETL(conn, params={"startDate": date_, "endDate": date_,
+                                                                      "date": date_.replace("-", "")})
+        try:
+            sd_report.reportV3_campaignMT_sdCampaigns_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
+        except:
+            conn = SD_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+            sd_report.reportV3_campaignMT_sdCampaigns_ETL(conn, params={"startDate": date_, "endDate": date_,
+                                                                      "date": date_.replace("-", "")})
+        try:
+            sd_report.reportV3_adgroup_sdAdGroup_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
+        except:
+            conn = SD_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+            sd_report.reportV3_adgroup_sdAdGroup_ETL(conn, params={"startDate": date_, "endDate": date_,
+                                                                      "date": date_.replace("-", "")})
+        try:
+            sd_report.reportV3_adgroupMT_sdAdGroup_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
+        except:
+            conn = SD_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+            sd_report.reportV3_adgroupMT_sdAdGroup_ETL(conn, params={"startDate": date_, "endDate": date_,
+                                                                      "date": date_.replace("-", "")})
+        try:
+            sd_report.reportV3_targeting_sdTargeting_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
+        except:
+            conn = SD_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+            sd_report.reportV3_targeting_sdTargeting_ETL(conn, params={"startDate": date_, "endDate": date_,
+                                                                      "date": date_.replace("-", "")})
+        try:
+            sd_report.reportV3_targetingMT_sdTargeting_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
+        except:
+            conn = SD_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+            sd_report.reportV3_targetingMT_sdTargeting_ETL(conn, params={"startDate": date_, "endDate": date_,
+                                                                      "date": date_.replace("-", "")})
+        try:
+            sd_report.reportV3_asin_sdPurchasedProduct_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
+        except:
+            conn = SD_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+            sd_report.reportV3_asin_sdPurchasedProduct_ETL(conn, params={"startDate": date_, "endDate": date_,
+                                                                      "date": date_.replace("-", "")})
+        try:
+            sd_report.reportV3_advertiser_sdAdvertisedProduct_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
+        except:
+            conn = SD_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+            sd_report.reportV3_advertiser_sdAdvertisedProduct_ETL(conn, params={"startDate": date_, "endDate": date_,
+                                                                      "date": date_.replace("-", "")})
 
         # sd_report.reportV2_campaignsRecord_t2_ETL(conn,params=para)
         # sd_report.reportV2_campaignsRecord_t3_ETL(conn,params=para)
@@ -147,6 +268,95 @@ def amz_report(AWS_CREDENTIALS,para=None):
         conn.close()
         time.sleep(5)
 
+def amz_report(AWS_CREDENTIALS,para=None):
+    count = 1
+    list_date = [((datetime.today()+timedelta(days=i)).date()).isoformat() for i in range(-2,-10,-1)]
+    conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+    # conn.command('')
+    if para is not None:
+        list_date = [para['startDate']]
+    # print(list_date)
+    delete_list = [
+                   'SB_sbAdGroup_adGroupV3',
+                   'SB_sbAds_adsV3',
+                   'SB_sbCampaigns_campaignV3',
+                   'SB_sbCampaigns_placementV3',
+                   'SB_sbPurchasedProduct_asinV3',
+                   'SB_sbSearchTerm_searchTermV3',
+                   'SB_sbTargeting_targetingV3',
+                   'SD_advertiser_sdAdvertisedProductV3',
+                   'SD_asin_sdPurchasedProductV3',
+                   'SD_sdAdGroup_adGroupMatchedTargetV3',
+                   'SD_sdAdGroup_adGroupV3',
+                   'SD_sdCampaigns_campaignMatchedTargetV3',
+                   'SD_sdCampaigns_campaignV3',
+                   'SD_targeting_sdTargetingMatchedTargetV3',
+                   'SD_targeting_sdTargetingV3',
+                   'SP_spAdvertisedProduct_advertiserV3',
+                   'SP_spCampaigns_adGroupV3',
+                   'SP_spCampaigns_campaignV3',
+                   'SP_spCampaigns_placementV3',
+                   'SP_spPurchasedProduct_asinV3',
+                   'SP_spSearchTerm_searchTermV3',
+                   'SP_spTargeting_targetingV3']
+    for date_ in list_date:
+        print(date_)
+        print(date_.replace("-", ""))
+        # amz_report(conn, AWS_CREDENTIALS, para={"startDate": date_, "endDate": date_, "date": date_.replace("-", "")})
+        if count==1:
+
+            for i in delete_list:
+                conn.command(f"delete from AmazonReport.{i} where date='{list_date[-2]}' and profileId='{AWS_CREDENTIALS['profile_id']}'")
+            count+=1
+        try:
+            refresh_token = shop_infos(AWS_CREDENTIALS['profile_id'])['refresh_token']
+            AWS_CREDENTIALS['refresh_token'] = refresh_token
+        except Exception as e:
+            print(e)
+
+        pre_func(AWS_CREDENTIALS,date_)
+
+    conn = SP_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+    date_list = []
+    for table_name in delete_list:
+        try:
+            locals()[table_name] = conn.query_df(
+                f"select date,SUM(impressions) as impressions from AmazonReport.`{table_name}` group by date,profileId")
+        except:
+            try:
+                locals()[table_name] = conn.query_df(
+                    f"select date,SUM(salesOtherSku1d) as salesOtherSku1d from AmazonReport.`{table_name}` group by date,profileId")
+            except:
+                try:
+                    locals()[table_name] = conn.query_df(
+                        f"select date,SUM(salesBrandHalo) as salesBrandHalo from AmazonReport.`{table_name}` group by date,profileId")
+                except:
+                    locals()[table_name] = conn.query_df(
+                        f"select date,SUM(sales14d) as sales14d  from AmazonReport.`{table_name}` group by date,profileId")
+        locals()[table_name].columns = ['date', 'refer_']
+        locals()[table_name]['refer_'] = locals()[table_name]['refer_'].astype('float64')
+        todayMinus2 = (datetime.today().date() + timedelta(days=-2))
+        reback_21days = (todayMinus2 + timedelta(days=-23)).isoformat()
+        df = pd.date_range(start=reback_21days, end=todayMinus2.isoformat(), freq='1d')
+        df1 = pd.DataFrame(df, columns=['date_'])
+        merge_df = df1.merge(locals()[table_name], left_on=['date_'], right_on=['date'], how='left')
+        rel = merge_df.query("date_!=date or refer_<1 or refer_.isna()")['date_'].map(
+            lambda x: x.date().isoformat()).tolist()
+        print(rel,date_list)
+        if len(rel)>0:
+            for date in rel:
+                date_list.extend([date])
+                conn.command(
+                    f"delete from AmazonReport.`{table_name}` where date='{date}' and profileId='{AWS_CREDENTIALS['profile_id']}'")
+    if len(date_list)>0:
+        print(list(set(date_list)))
+        for date_retry in list(set(date_list)):
+            try:
+                pre_func(AWS_CREDENTIALS, date_retry)
+            except:
+                pre_func(AWS_CREDENTIALS, date_retry)
+
+
 if __name__ == '__main__':
     # AccountTask("3006125408623189").do({"record": "portfolios"})
     AWS_CREDENTIALS = {

sync_amz_data/DataTransform/Data_ETL.py

@@ -246,7 +246,7 @@ class SP_ETL(SPClient, Common_ETLMethod):
         print("func_name:","reportV3_campaign_spCampaignsETL",'\n',"table_name:","SP_spCampaigns_campaignV3")
         if len(conn.query_df(f"select * from AmazonReport.SP_spCampaigns_campaignV3 where date='{params['startDate']}'"))>0:
             logging.info("数据已存在...")
-            time.sleep(5)
+            time.sleep(8.5)
             return 'Pass'
 
         params['reportType'] = "spCampaigns"
@@ -289,7 +289,7 @@ class SP_ETL(SPClient, Common_ETLMethod):
         if len(conn.query_df(
                 f"select * from AmazonReport.SP_spCampaigns_adGroupV3 where date='{params['startDate']}'")) > 0:
             logging.info("数据已存在...")
-            time.sleep(5)
+            time.sleep(8.5)
             return 'Pass'
         params['reportType'] = "spCampaigns"
         params['columns'] = [
@@ -332,7 +332,7 @@ class SP_ETL(SPClient, Common_ETLMethod):
         if len(conn.query_df(
                 f"select * from AmazonReport.SP_spCampaigns_placementV3 where date='{params['startDate']}'")) > 0:
             logging.info("数据已存在...")
-            time.sleep(5)
+            time.sleep(8.5)
             return 'Pass'
         params['columns'] = [
             'placementClassification','campaignName', 'campaignId','campaignStatus',  'campaignBudgetAmount', 'campaignBudgetType',
@@ -377,7 +377,7 @@ class SP_ETL(SPClient, Common_ETLMethod):
         if len(conn.query_df(
                 f"select * from AmazonReport.SP_spTargeting_targetingV3 where date='{params['startDate']}'")) > 0:
             logging.info("数据已存在...")
-            time.sleep(5)
+            time.sleep(8.5)
             return 'Pass'
         params['columns'] = [
             'adKeywordStatus',
@@ -422,7 +422,7 @@ class SP_ETL(SPClient, Common_ETLMethod):
         if len(conn.query_df(
                 f"select * from AmazonReport.SP_spSearchTerm_searchTermV3 where date='{params['startDate']}'")) > 0:
             logging.info("数据已存在...")
-            time.sleep(5)
+            time.sleep(8.5)
             return 'Pass'
         params['reportType'] = "spSearchTerm"
         params['columns'] = [
@@ -468,7 +468,7 @@ class SP_ETL(SPClient, Common_ETLMethod):
         if len(conn.query_df(
                 f"select * from AmazonReport.SP_spAdvertisedProduct_advertiserV3 where date='{params['startDate']}'")) > 0:
             logging.info("数据已存在...")
-            time.sleep(5)
+            time.sleep(8.5)
             return 'Pass'
         params['reportType'] = "spAdvertisedProduct"
         params['columns'] = [
@@ -516,7 +516,7 @@ class SP_ETL(SPClient, Common_ETLMethod):
             if len(conn.query_df(
                     f"select * from AmazonReport.SP_spPurchasedProduct_asinV3 where date='{params['startDate']}'")) > 0:
                 logging.info("数据已存在...")
-                time.sleep(5)
+                time.sleep(8.5)
                 return 'Pass'
         params['reportType'] = "spPurchasedProduct"
         params['columns'] = [
@@ -610,7 +610,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         if len(conn.query_df(
                 f"select * from AmazonReport.SB_sbCampaigns_campaignV3 where date='{params['startDate']}'")) > 0:
             logging.info("数据已存在...")
-            time.sleep(5)
+            time.sleep(8.5)
             return 'Pass'
         params['reportType'] = "sbCampaigns" #sbCampaigns
         params['columns'] = ['campaignName','campaignId','campaignStatus',
@@ -652,7 +652,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         if len(conn.query_df(
                 f"select * from AmazonReport.SB_sbAdGroup_adGroupV3 where date='{params['startDate']}'")) > 0:
             logging.info("数据已存在...")
-            time.sleep(5)
+            time.sleep(8.5)
             return 'Pass'
         params['reportType'] = "sbAdGroup" #sbCampaigns
         params['columns'] = ['campaignName','campaignId','campaignBudgetCurrencyCode','adGroupName','adGroupId', 'impressions',  'clicks', 'cost',
@@ -689,7 +689,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         if len(conn.query_df(
                 f"select * from AmazonReport.SB_sbCampaigns_placementV3 where date='{params['startDate']}'")) > 0:
             logging.info("数据已存在...")
-            time.sleep(5)
+            time.sleep(8.5)
             return 'Pass'
         params['reportType'] = "sbCampaignPlacement" #sbCampaigns
         params['columns'] = ['placementClassification','campaignName','campaignId','campaignStatus','campaignBudgetAmount','campaignBudgetType','campaignBudgetCurrencyCode',
@@ -727,7 +727,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         if len(conn.query_df(
                 f"select * from AmazonReport.SB_sbTargeting_targetingV3 where date='{params['startDate']}'")) > 0:
             logging.info("数据已存在...")
-            time.sleep(5)
+            time.sleep(8.5)
             return 'Pass'
         params['reportType'] = "sbTargeting" #sbCampaigns
         params['columns'] = [
@@ -767,7 +767,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         if len(conn.query_df(
                 f"select * from AmazonReport.SB_sbSearchTerm_searchTermV3 where date='{params['startDate']}'")) > 0:
             logging.info("数据已存在...")
-            time.sleep(5)
+            time.sleep(8.5)
             return 'Pass'
         params['reportType'] = "sbSearchTerm" #sbCampaigns
         params['columns'] = [
@@ -802,7 +802,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         if len(conn.query_df(
                 f"select * from AmazonReport.SB_sbAds_adsV3 where date='{params['startDate']}'")) > 0:
             logging.info("数据已存在...")
-            time.sleep(5)
+            time.sleep(8.5)
             return 'Pass'
         params['reportType'] = "sbAds" #sbCampaigns
         params['columns'] = [
@@ -845,7 +845,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         if len(conn.query_df(
                 f"select * from AmazonReport.SB_sbPurchasedProduct_asinV3 where date='{params['startDate']}'")) > 0:
             logging.info("数据已存在...")
-            time.sleep(5)
+            time.sleep(8.5)
             return 'Pass'
         params['reportType'] = "sbPurchasedProduct"
         params['columns'] = [
@@ -883,7 +883,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         if len(conn.query_df(
                 f"select * from AmazonReport.SB_campaignsV2 where date='{params['date']}'")) > 0:
             logging.info("数据已存在...")
-            time.sleep(5)
+            time.sleep(8.5)
             return 'Pass'
         params['record_type'] = 'campaigns'
 
@@ -923,7 +923,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         if len(conn.query_df(
                 f"select * from AmazonReport.SB_campaignsVideoV2 where date='{params['date']}'")) > 0:
             logging.info("数据已存在...")
-            time.sleep(5)
+            time.sleep(8.5)
             return 'Pass'
         params['record_type'] = 'campaigns'
 
@@ -965,7 +965,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         if len(conn.query_df(
                 f"select * from AmazonReport.SB_campaignsPlacementV2 where date='{params['date']}'")) > 0:
             logging.info("数据已存在...")
-            time.sleep(5)
+            time.sleep(8.5)
             return 'Pass'
         params['record_type'] = 'campaigns'
 
@@ -1002,7 +1002,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         if len(conn.query_df(
                 f"select * from AmazonReport.SB_campaignsPlacementVideoV2 where date='{params['date']}'")) > 0:
             logging.info("数据已存在...")
-            time.sleep(5)
+            time.sleep(8.5)
             return 'Pass'
         params['record_type'] = 'campaigns'
 
@@ -1042,7 +1042,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         if len(conn.query_df(
                 f"select * from AmazonReport.SB_adGroupsV2 where date='{params['date']}'")) > 0:
             logging.info("数据已存在...")
-            time.sleep(5)
+            time.sleep(8.5)
             return 'Pass'
         params['record_type'] = 'adGroups'
 
@@ -1082,7 +1082,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         if len(conn.query_df(
                 f"select * from AmazonReport.SB_adGroupsVideoV2 where date='{params['date']}'")) > 0:
             logging.info("数据已存在...")
-            time.sleep(5)
+            time.sleep(8.5)
             return 'Pass'
         metric = [
                 'campaignId','campaignName','adGroupId','adGroupName','impressions','clicks','cost',
@@ -1119,7 +1119,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         if len(conn.query_df(
                 f"select * from AmazonReport.SB_targetsV2 where date='{params['date']}'")) > 0:
             logging.info("数据已存在...")
-            time.sleep(5)
+            time.sleep(8.5)
             return 'Pass'
         params['record_type'] = 'targets'
 
@@ -1158,7 +1158,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         if len(conn.query_df(
                 f"select * from AmazonReport.SB_targetsVideoV2 where date='{params['date']}'")) > 0:
             logging.info("数据已存在...")
-            time.sleep(5)
+            time.sleep(8.5)
             return 'Pass'
         params['record_type'] = 'targets'
 
@@ -1198,7 +1198,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         if len(conn.query_df(
                 f"select * from AmazonReport.SB_keywordsV2 where date='{params['date']}'")) > 0:
             logging.info("数据已存在...")
-            time.sleep(5)
+            time.sleep(8.5)
             return 'Pass'
         params['record_type'] = 'keywords'
 
@@ -1237,7 +1237,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         if len(conn.query_df(
                 f"select * from AmazonReport.SB_keywordsVideoV2 where date='{params['date']}'")) > 0:
             logging.info("数据已存在...")
-            time.sleep(5)
+            time.sleep(8.5)
             return 'Pass'
         params['record_type'] = 'keywords'
 
@@ -1277,7 +1277,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         if len(conn.query_df(
                 f"select * from AmazonReport.SB_keywordsQueryV2 where date='{params['date']}'")) > 0:
             logging.info("数据已存在...")
-            time.sleep(5)
+            time.sleep(8.5)
             return 'Pass'
         params['record_type'] = 'keywords'
 
@@ -1311,7 +1311,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         if len(conn.query_df(
                 f"select * from AmazonReport.SB_keywordsQueryVideoV2 where date='{params['date']}'")) > 0:
             logging.info("数据已存在...")
-            time.sleep(5)
+            time.sleep(8.5)
             return 'Pass'
         params['record_type'] = 'keywords'
 
@@ -1348,7 +1348,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         if len(conn.query_df(
                 f"select * from AmazonReport.SB_adsV2 where date='{params['date']}'")) > 0:
             logging.info("数据已存在...")
-            time.sleep(5)
+            time.sleep(8.5)
             return 'Pass'
         params['record_type'] = 'ads'
 
@@ -1385,7 +1385,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         if len(conn.query_df(
                 f"select * from AmazonReport.SB_adsVideoV2 where date='{params['date']}'")) > 0:
             logging.info("数据已存在...")
-            time.sleep(5)
+            time.sleep(8.5)
             return 'Pass'
         params['record_type'] = 'ads'
 
@@ -1464,7 +1464,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         if len(conn.query_df(
                 f"select * from AmazonReport.SD_sdCampaigns_campaignV3 where date='{params['startDate']}'")) > 0:
             logging.info("数据已存在...")
-            time.sleep(5)
+            time.sleep(8.5)
             return 'Pass'
         params['reportType'] = "sdCampaigns"
         params['columns'] = [ 'campaignName', 'campaignId','campaignStatus','campaignBudgetAmount', 'impressions','clicks', 'cost',
@@ -1506,7 +1506,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         if len(conn.query_df(
                 f"select * from AmazonReport.SD_sdCampaigns_campaignMatchedTargetV3 where date='{params['startDate']}'")) > 0:
             logging.info("数据已存在...")
-            time.sleep(5)
+            time.sleep(8.5)
             return 'Pass'
         params['reportType'] = "sdCampaigns"
         params['columns'] = [
@@ -1548,7 +1548,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         if len(conn.query_df(
                 f"select * from AmazonReport.SD_sdAdGroup_adGroupV3 where date='{params['startDate']}'")) > 0:
             logging.info("数据已存在...")
-            time.sleep(5)
+            time.sleep(8.5)
             return 'Pass'
         params['reportType'] = "sdAdGroup"
         params['columns'] = ['campaignName','campaignId', 'adGroupName', 'adGroupId', 'impressions','clicks', 'cost',
@@ -1591,7 +1591,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         if len(conn.query_df(
                 f"select * from AmazonReport.SD_sdAdGroup_adGroupMatchedTargetV3 where date='{params['startDate']}'")) > 0:
             logging.info("数据已存在...")
-            time.sleep(5)
+            time.sleep(8.5)
             return 'Pass'
         params['reportType'] = "sdAdGroup"
         params['columns'] = [
@@ -1634,7 +1634,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         if len(conn.query_df(
                 f"select * from AmazonReport.SD_targeting_sdTargetingV3 where date='{params['startDate']}'")) > 0:
             logging.info("数据已存在...")
-            time.sleep(5)
+            time.sleep(8.5)
             return 'Pass'
         params['reportType'] = "sdTargeting"
         params['columns'] = ['campaignName', 'campaignId','adGroupName','adGroupId', 'targetingText','targetingId','impressions','clicks', 'cost',
@@ -1681,7 +1681,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         if len(conn.query_df(
                 f"select * from AmazonReport.SD_targeting_sdTargetingMatchedTargetV3 where date='{params['startDate']}'")) > 0:
             logging.info("数据已存在...")
-            time.sleep(5)
+            time.sleep(8.5)
             return 'Pass'
         params['reportType'] = "sdTargeting"
         params['columns'] = [
@@ -1729,7 +1729,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         if len(conn.query_df(
                 f"select * from AmazonReport.SD_advertiser_sdAdvertisedProductV3 where date='{params['startDate']}'")) > 0:
             logging.info("数据已存在...")
-            time.sleep(5)
+            time.sleep(8.5)
             return 'Pass'
         params['reportType'] = "sdAdvertisedProduct"
         params['columns'] = ['campaignName','campaignId','adGroupName','adGroupId','adId','impressions','clicks', 'cost',
@@ -1774,7 +1774,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         if len(conn.query_df(
                 f"select * from AmazonReport.SD_asin_sdPurchasedProductV3 where date='{params['startDate']}'")) > 0:
             logging.info("数据已存在...")
-            time.sleep(5)
+            time.sleep(8.5)
             return 'Pass'
         params['reportType'] = "sdPurchasedProduct"
         params['columns'] = ['campaignName','campaignId', 'adGroupName','adGroupId', 'promotedAsin', 'promotedSku',
@@ -1828,7 +1828,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         if len(conn.query_df(
                 f"select * from AmazonReport.SD_campaignsV2 where date='{params['date']}'")) > 0:
             logging.info("数据已存在...")
-            time.sleep(5)
+            time.sleep(8.5)
             return 'Pass'
         params['record_type'] = 'campaigns'
 
@@ -1858,7 +1858,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         if len(conn.query_df(
                 f"select * from AmazonReport.SD_campaignsV2 where date='{params['date']}'")) > 0:
             logging.info("数据已存在...")
-            time.sleep(5)
+            time.sleep(8.5)
             return 'Pass'
         params['record_type'] = 'campaigns'
 
@@ -1905,7 +1905,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         if len(conn.query_df(
                 f"select * from AmazonReport.SD_adGroupsV2 where date='{params['date']}'")) > 0:
             logging.info("数据已存在...")
-            time.sleep(5)
+            time.sleep(8.5)
             return 'Pass'
         params['record_type'] = 'adGroups'
 
@@ -1933,7 +1933,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         if len(conn.query_df(
                 f"select * from AmazonReport.SD_adGroupsV2 where date='{params['date']}'")) > 0:
             logging.info("数据已存在...")
-            time.sleep(5)
+            time.sleep(8.5)
             return 'Pass'
 
         params['record_type'] = 'adGroups'
@@ -1978,7 +1978,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         if len(conn.query_df(
                 f"select * from AmazonReport.SD_adsV2 where date='{params['date']}'")) > 0:
             logging.info("数据已存在...")
-            time.sleep(5)
+            time.sleep(8.5)
             return 'Pass'
         params['record_type'] = 'productAds'
 
@@ -2007,7 +2007,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         if len(conn.query_df(
                 f"select * from AmazonReport.SD_adsV2 where date='{params['date']}'")) > 0:
             logging.info("数据已存在...")
-            time.sleep(5)
+            time.sleep(8.5)
             return 'Pass'
         params['record_type'] = 'productAds'
 
@@ -2052,7 +2052,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         if len(conn.query_df(
                 f"select * from AmazonReport.SD_targetsV2 where date='{params['date']}'")) > 0:
             logging.info("数据已存在...")
-            time.sleep(5)
+            time.sleep(8.5)
             return 'Pass'
         params['record_type'] = 'targets'
 
@@ -2081,7 +2081,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         if len(conn.query_df(
                 f"select * from AmazonReport.SD_targetsV2 where date='{params['date']}'")) > 0:
             logging.info("数据已存在...")
-            time.sleep(5)
+            time.sleep(8.5)
             return 'Pass'
         params['record_type'] = 'targets'
 
@@ -2123,7 +2123,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         if len(conn.query_df(
                 f"select * from AmazonReport.SD_asinsV2 where date='{params['date']}'")) > 0:
             logging.info("数据已存在...")
-            time.sleep(5)
+            time.sleep(8.5)
             return 'Pass'
         params['record_type'] = 'asins'
 
@@ -2191,7 +2191,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         if len(conn.query_df(
                 f"select * from AmazonReport.SD_campaignsMatchedTargetV2 where date='{params['date']}'")) > 0:
             logging.info("数据已存在...")
-            time.sleep(5)
+            time.sleep(8.5)
             return 'Pass'
 
         params['record_type'] = 'campaigns'
@@ -2223,7 +2223,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         if len(conn.query_df(
                 f"select * from AmazonReport.SD_campaignsMatchedTargetV2 where date='{params['date']}'")) > 0:
             logging.info("数据已存在...")
-            time.sleep(5)
+            time.sleep(8.5)
             return 'Pass'
         params['record_type'] = 'campaigns'
 
@@ -2267,7 +2267,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         if len(conn.query_df(
                 f"select * from AmazonReport.SD_adGroupsMatchedTargetV2 where date='{params['date']}'")) > 0:
             logging.info("数据已存在...")
-            time.sleep(5)
+            time.sleep(8.5)
             return 'Pass'
         params['record_type'] = 'adGroups'
 
@@ -2296,7 +2296,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         if len(conn.query_df(
                 f"select * from AmazonReport.SD_adGroupsMatchedTargetV2 where date='{params['date']}'")) > 0:
             logging.info("数据已存在...")
-            time.sleep(5)
+            time.sleep(8.5)
             return 'Pass'
         params['record_type'] = 'adGroups'
 
@@ -2338,7 +2338,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         if len(conn.query_df(
                 f"select * from AmazonReport.SD_targetsMatchedTargetV2 where date='{params['date']}'")) > 0:
             logging.info("数据已存在...")
-            time.sleep(5)
+            time.sleep(8.5)
             return 'Pass'
         params['record_type'] = 'targets'
 
@@ -2366,7 +2366,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         if len(conn.query_df(
                 f"select * from AmazonReport.SD_targetsMatchedTargetV2 where date='{params['date']}'")) > 0:
             logging.info("数据已存在...")
-            time.sleep(5)
+            time.sleep(8.5)
             return 'Pass'
         params['record_type'] = 'targets'
 

sync_get_order_data.py

@@ -16,7 +16,7 @@ def func_run():
         print(e)
     try:
         count = 0
-        for days in range(-2,-10,-1): #range(-2,-72,-1): #
+        for days in range(-2,-10,-1): #range(-58,-77,-1): #
             if count in (0,1,2):#[i for i in range(0,32)]:
                 conn = sp_api_client.SpApiRequest.Data_auth()
                 cursor = conn.cursor()

sync_listing_order_Retry.py

@@ -3,7 +3,7 @@ warnings.filterwarnings('ignore')
 from apscheduler.schedulers.blocking import BlockingScheduler
 from sync_amz_data.public import sp_api_client
 def func_run():
-    days = -2
+    days = -4
     try:
         sp_api_client.SpApiRequest.get_allShops("GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL",days=days,**{})
     except Exception as e: