sync modify

start_sync_amz.py

@@ -11,272 +11,149 @@ from apscheduler.schedulers.blocking import BlockingScheduler
 from datetime import datetime,timedelta
 
 def pre_func(AWS_CREDENTIALS,date_):
+    conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+    sb_report = SB_ETL(**AWS_CREDENTIALS)
+    sb_report.reportV3_campaign_sbCampaigns_ETL(conn, params={"startDate": date_, "endDate": date_,
+                                                              "date": date_.replace("-", "")})
 
+    sb_report.reportV3_adGroup_sbAdGroup_ETL(conn, params={"startDate": date_, "endDate": date_,
+                                                           "date": date_.replace("-", "")})
 
-        conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
-        sb_report = SB_ETL(**AWS_CREDENTIALS)
-        try:
-            sb_report.reportV3_campaign_sbCampaigns_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        except:
-            conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
-            sb_report.reportV3_campaign_sbCampaigns_ETL(conn, params={"startDate": date_, "endDate": date_,
-                                                                      "date": date_.replace("-", "")})
-        try:
-            sb_report.reportV3_adGroup_sbAdGroup_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        except:
-            conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
-            sb_report.reportV3_adGroup_sbAdGroup_ETL(conn, params={"startDate": date_, "endDate": date_,
-                                                                      "date": date_.replace("-", "")})
-        try:
-            sb_report.reportV3_sbCampaignPlacement_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        except:
-            conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
-            sb_report.reportV3_sbCampaignPlacement_ETL(conn, params={"startDate": date_, "endDate": date_,
-                                                                      "date": date_.replace("-", "")})
-        try:
-            sb_report.reportV3_sbTargeting_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        except:
-            conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
-            sb_report.reportV3_sbTargeting_ETL(conn, params={"startDate": date_, "endDate": date_,
-                                                                      "date": date_.replace("-", "")})
-        try:
-            sb_report.reportV3_sbSearchTerm_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        except:
-            conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
-            sb_report.reportV3_sbSearchTerm_ETL(conn, params={"startDate": date_, "endDate": date_,
-                                                                      "date": date_.replace("-", "")})
-        try:
-            sb_report.reportV3_sbAds_ETL(conn, params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        except:
-            conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
-            sb_report.reportV3_sbAds_ETL(conn, params={"startDate": date_, "endDate": date_,
-                                                                      "date": date_.replace("-", "")})
-        try:
-            sb_report.reportV3_purchasedAsinRecord_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        except:
-            conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
-            sb_report.reportV3_purchasedAsinRecord_ETL(conn, params={"startDate": date_, "endDate": date_,
-                                                                      "date": date_.replace("-", "")})
-        try:
-            sb_report.reportV3_purchasedAsinRecord_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        except:
-            conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
-            sb_report.reportV3_purchasedAsinRecord_ETL(conn, params={"startDate": date_, "endDate": date_,
-                                                                      "date": date_.replace("-", "")})
-        try:
-            sb_report.reportV2_campaignsRecord_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        except:
-            conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
-            sb_report.reportV2_campaignsRecord_ETL(conn, params={"startDate": date_, "endDate": date_,
-                                                                      "date": date_.replace("-", "")})
-        try:
-            sb_report.reportV2_campaignsVideo_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        except:
-            conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
-            sb_report.reportV2_campaignsVideo_ETL(conn, params={"startDate": date_, "endDate": date_,
-                                                                      "date": date_.replace("-", "")})
-        try:
-            sb_report.reportV2_adGroupsRecord_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        except:
-            conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
-            sb_report.reportV2_adGroupsRecord_ETL(conn, params={"startDate": date_, "endDate": date_,
-                                                                      "date": date_.replace("-", "")})
+    sb_report.reportV3_adGroup_sbAdGroup_ETL(conn, params={"startDate": date_, "endDate": date_,
+                                                           "date": date_.replace("-", "")})
 
-        try:
-            sb_report.reportV2_adGroupsVideo_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        except:
-            conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
-            sb_report.reportV2_adGroupsVideo_ETL(conn, params={"startDate": date_, "endDate": date_,
-                                                                      "date": date_.replace("-", "")})
-        try:
-            sb_report.reportV2_adsRecord_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        except:
-            conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
-            sb_report.reportV2_adsRecord_ETL(conn, params={"startDate": date_, "endDate": date_,
-                                                                      "date": date_.replace("-", "")})
-        try:
-            sb_report.reportV2_adsVideo_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        except:
-            conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
-            sb_report.reportV2_adsVideo_ETL(conn, params={"startDate": date_, "endDate": date_,
-                                                                      "date": date_.replace("-", "")})
-        try:
-            sb_report.reportV2_keywordsRecord_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        except:
-            conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
-            sb_report.reportV2_keywordsRecord_ETL(conn, params={"startDate": date_, "endDate": date_,
-                                                                      "date": date_.replace("-", "")})
-        try:
-            sb_report.reportV2_keywordsVideo_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        except:
-            conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
-            sb_report.reportV2_keywordsVideo_ETL(conn, params={"startDate": date_, "endDate": date_,
-                                                                      "date": date_.replace("-", "")})
-        try:
-            sb_report.reportV2_placementRecord_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        except:
-            conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
-            sb_report.reportV2_placementRecord_ETL(conn, params={"startDate": date_, "endDate": date_,
-                                                                      "date": date_.replace("-", "")})
-        try:
-            sb_report.reportV2_placementVideo_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        except:
-            conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
-            sb_report.reportV2_placementVideo_ETL(conn, params={"startDate": date_, "endDate": date_,
-                                                                      "date": date_.replace("-", "")})
-        try:
-            sb_report.reportV2_searchtermsRecord_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        except:
-            conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
-            sb_report.reportV2_searchtermsRecord_ETL(conn, params={"startDate": date_, "endDate": date_,
-                                                                      "date": date_.replace("-", "")})
-        try:
-            sb_report.reportV2_searchtermsVideo_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        except:
-            conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
-            sb_report.reportV2_searchtermsVideo_ETL(conn, params={"startDate": date_, "endDate": date_,
-                                                                      "date": date_.replace("-", "")})
-        try:
-            sb_report.reportV2_targetsRecord_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        except:
-            conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
-            sb_report.reportV2_targetsRecord_ETL(conn, params={"startDate": date_, "endDate": date_,
-                                                                      "date": date_.replace("-", "")})
-        try:
-            sb_report.reportV2_targetsVideo_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        except:
-            conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
-            sb_report.reportV2_targetsVideo_ETL(conn, params={"startDate": date_, "endDate": date_,
-                                                                      "date": date_.replace("-", "")})
+    sb_report.reportV3_sbCampaignPlacement_ETL(conn, params={"startDate": date_, "endDate": date_,
+                                                             "date": date_.replace("-", "")})
 
-        conn.close()
-        time.sleep(5)
+    sb_report.reportV3_sbTargeting_ETL(conn,
+                                       params={"startDate": date_, "endDate": date_, "date": date_.replace("-", "")})
 
-        conn = SP_ETL(**AWS_CREDENTIALS).clickhouse_connect()
-        sp_report = SP_ETL(**AWS_CREDENTIALS)
-        try:
-            sp_report.reportV3_campaign_spCampaignsETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        except:
-            conn = SP_ETL(**AWS_CREDENTIALS).clickhouse_connect()
-            sp_report.reportV3_campaign_spCampaignsETL(conn, params={"startDate": date_, "endDate": date_,
-                                                                      "date": date_.replace("-", "")})
-        try:
-            sp_report.reportV3_adGroup_spCampaignsETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        except:
-            conn = SP_ETL(**AWS_CREDENTIALS).clickhouse_connect()
-            sp_report.reportV3_adGroup_spCampaignsETL(conn, params={"startDate": date_, "endDate": date_,
-                                                                      "date": date_.replace("-", "")})
-        try:
-            sp_report.reportV3_campaignPlacement_spCampaignsETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        except:
-            conn = SP_ETL(**AWS_CREDENTIALS).clickhouse_connect()
-            sp_report.reportV3_campaignPlacement_spCampaignsETL(conn, params={"startDate": date_, "endDate": date_,
-                                                                      "date": date_.replace("-", "")})
-        try:
-            sp_report.reportV3_targeting_spTargetingETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        except:
-            conn = SP_ETL(**AWS_CREDENTIALS).clickhouse_connect()
-            sp_report.reportV3_targeting_spTargetingETL(conn, params={"startDate": date_, "endDate": date_,
-                                                                      "date": date_.replace("-", "")})
-        try:
-            sp_report.reportV3_searchTerm_spSearchTermETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        except:
-            conn = SP_ETL(**AWS_CREDENTIALS).clickhouse_connect()
-            sp_report.reportV3_searchTerm_spSearchTermETL(conn, params={"startDate": date_, "endDate": date_,
-                                                                      "date": date_.replace("-", "")})
-        try:
-            sp_report.reportV3_advertiser_spAdvertisedProductETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        except:
-            conn = SP_ETL(**AWS_CREDENTIALS).clickhouse_connect()
-            sp_report.reportV3_advertiser_spAdvertisedProductETL(conn, params={"startDate": date_, "endDate": date_,
-                                                                      "date": date_.replace("-", "")})
-        try:
-            sp_report.reportV3_asin_spPurchasedProductETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        except:
-            conn = SP_ETL(**AWS_CREDENTIALS).clickhouse_connect()
-            sp_report.reportV3_asin_spPurchasedProductETL(conn, params={"startDate": date_, "endDate": date_,
-                                                                      "date": date_.replace("-", "")})
-        conn.close()
-        time.sleep(5)
+    sb_report.reportV3_sbSearchTerm_ETL(conn,
+                                        params={"startDate": date_, "endDate": date_, "date": date_.replace("-", "")})
 
+    sb_report.reportV3_sbAds_ETL(conn, params={"startDate": date_, "endDate": date_, "date": date_.replace("-", "")})
 
-        conn = SD_ETL(**AWS_CREDENTIALS).clickhouse_connect()
-        sd_report = SD_ETL(**AWS_CREDENTIALS)
-        try:
-            sd_report.reportV3_campaign_sdCampaigns_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        except:
-            conn = SD_ETL(**AWS_CREDENTIALS).clickhouse_connect()
-            sd_report.reportV3_campaign_sdCampaigns_ETL(conn, params={"startDate": date_, "endDate": date_,
-                                                                      "date": date_.replace("-", "")})
-        try:
-            sd_report.reportV3_campaignMT_sdCampaigns_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        except:
-            conn = SD_ETL(**AWS_CREDENTIALS).clickhouse_connect()
-            sd_report.reportV3_campaignMT_sdCampaigns_ETL(conn, params={"startDate": date_, "endDate": date_,
-                                                                      "date": date_.replace("-", "")})
-        try:
-            sd_report.reportV3_adgroup_sdAdGroup_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        except:
-            conn = SD_ETL(**AWS_CREDENTIALS).clickhouse_connect()
-            sd_report.reportV3_adgroup_sdAdGroup_ETL(conn, params={"startDate": date_, "endDate": date_,
-                                                                      "date": date_.replace("-", "")})
-        try:
-            sd_report.reportV3_adgroupMT_sdAdGroup_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        except:
-            conn = SD_ETL(**AWS_CREDENTIALS).clickhouse_connect()
-            sd_report.reportV3_adgroupMT_sdAdGroup_ETL(conn, params={"startDate": date_, "endDate": date_,
-                                                                      "date": date_.replace("-", "")})
-        try:
-            sd_report.reportV3_targeting_sdTargeting_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        except:
-            conn = SD_ETL(**AWS_CREDENTIALS).clickhouse_connect()
-            sd_report.reportV3_targeting_sdTargeting_ETL(conn, params={"startDate": date_, "endDate": date_,
-                                                                      "date": date_.replace("-", "")})
-        try:
-            sd_report.reportV3_targetingMT_sdTargeting_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        except:
-            conn = SD_ETL(**AWS_CREDENTIALS).clickhouse_connect()
-            sd_report.reportV3_targetingMT_sdTargeting_ETL(conn, params={"startDate": date_, "endDate": date_,
-                                                                      "date": date_.replace("-", "")})
-        try:
-            sd_report.reportV3_asin_sdPurchasedProduct_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        except:
-            conn = SD_ETL(**AWS_CREDENTIALS).clickhouse_connect()
-            sd_report.reportV3_asin_sdPurchasedProduct_ETL(conn, params={"startDate": date_, "endDate": date_,
-                                                                      "date": date_.replace("-", "")})
-        try:
-            sd_report.reportV3_advertiser_sdAdvertisedProduct_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        except:
-            conn = SD_ETL(**AWS_CREDENTIALS).clickhouse_connect()
-            sd_report.reportV3_advertiser_sdAdvertisedProduct_ETL(conn, params={"startDate": date_, "endDate": date_,
+    sb_report.reportV3_purchasedAsinRecord_ETL(conn, params={"startDate": date_, "endDate": date_,
+                                                             "date": date_.replace("-", "")})
+
+    sb_report.reportV3_purchasedAsinRecord_ETL(conn, params={"startDate": date_, "endDate": date_,
+                                                             "date": date_.replace("-", "")})
+
+    sb_report.reportV2_campaignsRecord_ETL(conn, params={"startDate": date_, "endDate": date_,
+                                                         "date": date_.replace("-", "")})
+
+    sb_report.reportV2_campaignsVideo_ETL(conn,
+                                          params={"startDate": date_, "endDate": date_, "date": date_.replace("-", "")})
+
+    sb_report.reportV2_adGroupsRecord_ETL(conn,
+                                          params={"startDate": date_, "endDate": date_, "date": date_.replace("-", "")})
+
+    sb_report.reportV2_adGroupsVideo_ETL(conn,
+                                         params={"startDate": date_, "endDate": date_, "date": date_.replace("-", "")})
+
+    sb_report.reportV2_adsRecord_ETL(conn,
+                                     params={"startDate": date_, "endDate": date_, "date": date_.replace("-", "")})
+
+    sb_report.reportV2_adsVideo_ETL(conn, params={"startDate": date_, "endDate": date_, "date": date_.replace("-", "")})
+
+    sb_report.reportV2_keywordsRecord_ETL(conn,
+                                          params={"startDate": date_, "endDate": date_, "date": date_.replace("-", "")})
+
+    sb_report.reportV2_keywordsVideo_ETL(conn,
+                                         params={"startDate": date_, "endDate": date_, "date": date_.replace("-", "")})
+
+    sb_report.reportV2_placementRecord_ETL(conn, params={"startDate": date_, "endDate": date_,
+                                                         "date": date_.replace("-", "")})
+
+    sb_report.reportV2_placementVideo_ETL(conn,
+                                          params={"startDate": date_, "endDate": date_, "date": date_.replace("-", "")})
+
+    sb_report.reportV2_searchtermsRecord_ETL(conn, params={"startDate": date_, "endDate": date_,
+                                                           "date": date_.replace("-", "")})
+
+    sb_report.reportV2_searchtermsVideo_ETL(conn, params={"startDate": date_, "endDate": date_,
+                                                          "date": date_.replace("-", "")})
+
+    sb_report.reportV2_targetsRecord_ETL(conn,
+                                         params={"startDate": date_, "endDate": date_, "date": date_.replace("-", "")})
+
+    sb_report.reportV2_targetsVideo_ETL(conn,
+                                        params={"startDate": date_, "endDate": date_, "date": date_.replace("-", "")})
+
+    conn.close()
+    time.sleep(5)
+
+    conn = SP_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+    sp_report = SP_ETL(**AWS_CREDENTIALS)
+
+    sp_report.reportV3_campaign_spCampaignsETL(conn, params={"startDate": date_, "endDate": date_,
+                                                             "date": date_.replace("-", "")})
+
+    sp_report.reportV3_adGroup_spCampaignsETL(conn, params={"startDate": date_, "endDate": date_,
+                                                            "date": date_.replace("-", "")})
+
+    sp_report.reportV3_campaignPlacement_spCampaignsETL(conn, params={"startDate": date_, "endDate": date_,
                                                                       "date": date_.replace("-", "")})
 
-        # sd_report.reportV2_campaignsRecord_t2_ETL(conn,params=para)
-        # sd_report.reportV2_campaignsRecord_t3_ETL(conn,params=para)
-        # sd_report.reportV2_adGroupsRecord_t2_ETL(conn,params=para)
-        # sd_report.reportV2_adGroupsRecord_t3_ETL(conn,params=para)
-        # sd_report.reportV2_asins_t2_ETL(conn,params=para)
+    sp_report.reportV3_targeting_spTargetingETL(conn, params={"startDate": date_, "endDate": date_,
+                                                              "date": date_.replace("-", "")})
+
+    sp_report.reportV3_searchTerm_spSearchTermETL(conn, params={"startDate": date_, "endDate": date_,
+                                                                "date": date_.replace("-", "")})
+
+    sp_report.reportV3_advertiser_spAdvertisedProductETL(conn, params={"startDate": date_, "endDate": date_,
+                                                                       "date": date_.replace("-", "")})
+
+    sp_report.reportV3_asin_spPurchasedProductETL(conn, params={"startDate": date_, "endDate": date_,
+                                                                "date": date_.replace("-", "")})
 
+    conn.close()
+    time.sleep(5)
 
+    conn = SD_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+    sd_report = SD_ETL(**AWS_CREDENTIALS)
 
-        # sd_report.reportV2_asins_t3_ETL(conn,params=para)
-        # sd_report.reportV2_productAds_t2_ETL(conn,params=para)
-        # sd_report.reportV2_productAds_t3_ETL(conn,params=para)
-        # sd_report.reportV2_targets_t2_ETL(conn,params=para)
-        # sd_report.reportV2_targets_t3_ETL(conn,params=para)
-        # sd_report.reportV2_campaign_matchedTarget_t2_ETL(conn,params=para)
-        # sd_report.reportV2_campaign_matchedTarget_t3_ETL(conn,params=para)
-        # sd_report.reportV2_adGroups_matchedTarget_t2_ETL(conn,params=para)
-        # sd_report.reportV2_adGroups_matchedTarget_t3_ETL(conn,params=para)
-        # sd_report.reportV2_targets_matchedTarget_t2_ETL(conn,params=para)
-        # sd_report.reportV2_targets_matchedTarget_t3_ETL(conn,params=para)
-        conn.close()
-        time.sleep(5)
+    sd_report.reportV3_campaign_sdCampaigns_ETL(conn, params={"startDate": date_, "endDate": date_,
+                                                              "date": date_.replace("-", "")})
+    sd_report.reportV3_campaignMT_sdCampaigns_ETL(conn, params={"startDate": date_, "endDate": date_,
+                                                                "date": date_.replace("-", "")})
+    sd_report.reportV3_adgroup_sdAdGroup_ETL(conn, params={"startDate": date_, "endDate": date_,
+                                                           "date": date_.replace("-", "")})
+    sd_report.reportV3_adgroupMT_sdAdGroup_ETL(conn, params={"startDate": date_, "endDate": date_,
+                                                             "date": date_.replace("-", "")})
+    sd_report.reportV3_targeting_sdTargeting_ETL(conn, params={"startDate": date_, "endDate": date_,
+                                                               "date": date_.replace("-", "")})
+    sd_report.reportV3_targetingMT_sdTargeting_ETL(conn, params={"startDate": date_, "endDate": date_,
+                                                                 "date": date_.replace("-", "")})
+    sd_report.reportV3_asin_sdPurchasedProduct_ETL(conn, params={"startDate": date_, "endDate": date_,
+                                                                 "date": date_.replace("-", "")})
+    sd_report.reportV3_advertiser_sdAdvertisedProduct_ETL(conn, params={"startDate": date_, "endDate": date_,
+                                                                        "date": date_.replace("-", "")})
+
+    # sd_report.reportV2_campaignsRecord_t2_ETL(conn,params=para)
+    # sd_report.reportV2_campaignsRecord_t3_ETL(conn,params=para)
+    # sd_report.reportV2_adGroupsRecord_t2_ETL(conn,params=para)
+    # sd_report.reportV2_adGroupsRecord_t3_ETL(conn,params=para)
+    # sd_report.reportV2_asins_t2_ETL(conn,params=para)
+
+    # sd_report.reportV2_asins_t3_ETL(conn,params=para)
+    # sd_report.reportV2_productAds_t2_ETL(conn,params=para)
+    # sd_report.reportV2_productAds_t3_ETL(conn,params=para)
+    # sd_report.reportV2_targets_t2_ETL(conn,params=para)
+    # sd_report.reportV2_targets_t3_ETL(conn,params=para)
+    # sd_report.reportV2_campaign_matchedTarget_t2_ETL(conn,params=para)
+    # sd_report.reportV2_campaign_matchedTarget_t3_ETL(conn,params=para)
+    # sd_report.reportV2_adGroups_matchedTarget_t2_ETL(conn,params=para)
+    # sd_report.reportV2_adGroups_matchedTarget_t3_ETL(conn,params=para)
+    # sd_report.reportV2_targets_matchedTarget_t2_ETL(conn,params=para)
+    # sd_report.reportV2_targets_matchedTarget_t3_ETL(conn,params=para)
+    conn.close()
+    time.sleep(5)
 
 def amz_report(AWS_CREDENTIALS,para=None):
     count = 1
-    list_date = [((datetime.today()+timedelta(days=i)).date()).isoformat() for i in range(-2,-20,-1)]
-    conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+    list_date = [((datetime.today()+timedelta(days=i)).date()).isoformat() for i in range(-2,-8,-1)]
+    # conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
     # conn.command('')
     if para is not None:
         list_date = [para['startDate']]
@@ -304,16 +181,17 @@ def amz_report(AWS_CREDENTIALS,para=None):
                    'SP_spPurchasedProduct_asinV3',
                    'SP_spSearchTerm_searchTermV3',
                    'SP_spTargeting_targetingV3']
+    # conn = SP_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+    # for i in delete_list:
+    #     conn.command(f"delete from AmazonReport.{i} where date='{(datetime.today()-timedelta(days=8)).date().isoformat()}' and profileId='{AWS_CREDENTIALS['profile_id']}'")
 
     for date_ in list_date:
         print(date_)
         print(date_.replace("-", ""))
-        # amz_report(conn, AWS_CREDENTIALS, para={"startDate": date_, "endDate": date_, "date": date_.replace("-", "")})
-        if count==1:
 
-            for i in delete_list:
-                conn.command(f"delete from AmazonReport.{i} where date='{list_date[-2]}' and profileId='{AWS_CREDENTIALS['profile_id']}'")
-            count+=1
+        #     for i in delete_list:
+        #         conn.command(f"delete from AmazonReport.{i} where date='{list_date[-2]}' and profileId='{AWS_CREDENTIALS['profile_id']}'")
+
         try:
             refresh_token = shop_infos(AWS_CREDENTIALS['profile_id'])['refresh_token']
             AWS_CREDENTIALS['refresh_token'] = refresh_token
@@ -321,87 +199,65 @@ def amz_report(AWS_CREDENTIALS,para=None):
         except Exception as e:
             print(e)
 
-        pre_func(AWS_CREDENTIALS,date_)
-    try:
-        conn = SP_ETL(**AWS_CREDENTIALS).clickhouse_connect()
-        date_list = []
-        for table_name in delete_list:
+            pre_func(AWS_CREDENTIALS,date_)
+
+    if datetime.today().weekday() in [2,6]:
+        print('execute func2')
+        try:
+            date_list = func_2(delete_list)
+            func_3(date_list)
+        except Exception as e:
+            logging.warning(e)
+            date_list = func_2(delete_list)
+            func_3(date_list)
+
+def func_2(delete_list):
+    conn = SP_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+    date_list = []
+    for table_name in delete_list:
+        check_date = (datetime.today() + timedelta(days=14)).date().isoformat()
+        try:
+            locals()[table_name] = conn.query_df(
+                f"select date,SUM(impressions) as impressions from AmazonReport.`{table_name}` where date >='{check_date}' group by date,profileId")
+        except:
             try:
                 locals()[table_name] = conn.query_df(
-                    f"select date,SUM(impressions) as impressions from AmazonReport.`{table_name}` group by date,profileId")
+                    f"select date,SUM(salesOtherSku1d) as salesOtherSku1d from AmazonReport.`{table_name}` date >='{check_date}' group by date,profileId")
             except:
                 try:
                     locals()[table_name] = conn.query_df(
-                        f"select date,SUM(salesOtherSku1d) as salesOtherSku1d from AmazonReport.`{table_name}` group by date,profileId")
+                        f"select date,SUM(salesBrandHalo) as salesBrandHalo from AmazonReport.`{table_name}` date >='{check_date}' group by date,profileId")
                 except:
-                    try:
-                        locals()[table_name] = conn.query_df(
-                            f"select date,SUM(salesBrandHalo) as salesBrandHalo from AmazonReport.`{table_name}` group by date,profileId")
-                    except:
-                        locals()[table_name] = conn.query_df(
-                            f"select date,SUM(sales14d) as sales14d  from AmazonReport.`{table_name}` group by date,profileId")
-            locals()[table_name].columns = ['date', 'refer_']
-            locals()[table_name]['refer_'] = locals()[table_name]['refer_'].astype('float64')
-            todayMinus2 = (datetime.today().date() + timedelta(days=-2))
-            reback_21days = (todayMinus2 + timedelta(days=-23)).isoformat()
-            df = pd.date_range(start=reback_21days, end=todayMinus2.isoformat(), freq='1d')
-            df1 = pd.DataFrame(df, columns=['date_'])
-            merge_df = df1.merge(locals()[table_name], left_on=['date_'], right_on=['date'], how='left')
-            rel = merge_df.query("date_!=date or refer_<1 or refer_.isna()")['date_'].map(
-                lambda x: x.date().isoformat()).tolist()
-            print(rel,date_list)
-            if len(rel)>0:
-                for date in rel:
-                    date_list.extend([date])
-                    conn.command(
-                        f"delete from AmazonReport.`{table_name}` where date='{date}' and profileId='{AWS_CREDENTIALS['profile_id']}'")
-        if len(date_list)>0:
-            print(list(set(date_list)))
-            for date_retry in list(set(date_list)):
-                try:
-                    pre_func(AWS_CREDENTIALS, date_retry)
-                except:
-                    pre_func(AWS_CREDENTIALS, date_retry)
-    except:
-        conn = SP_ETL(**AWS_CREDENTIALS).clickhouse_connect()
-        date_list = []
-        for table_name in delete_list:
+                    locals()[table_name] = conn.query_df(
+                        f"select date,SUM(sales14d) as sales14d  from AmazonReport.`{table_name}` date >='{check_date}' group by date,profileId")
+        locals()[table_name].columns = ['date', 'refer_']
+        locals()[table_name]['refer_'] = locals()[table_name]['refer_'].astype('float64')
+        todayMinus2 = (datetime.today().date() + timedelta(days=-2))
+        reback_21days = (todayMinus2 + timedelta(days=-8)).isoformat()
+        df = pd.date_range(start=reback_21days, end=todayMinus2.isoformat(), freq='1d')
+        df1 = pd.DataFrame(df, columns=['date_'])
+        merge_df = df1.merge(locals()[table_name], left_on=['date_'], right_on=['date'], how='left')
+        rel = merge_df.query("date_!=date or refer_<1 or refer_.isna()")['date_'].map(
+            lambda x: x.date().isoformat()).tolist()
+        print(rel, date_list)
+        if len(rel) > 0:
+            for date in rel:
+                date_list.extend([date])
+                conn = SP_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+                conn.command(
+                    f"delete from AmazonReport.`{table_name}` where date='{date}' and profileId='{AWS_CREDENTIALS['profile_id']}'")
+
+    return date_list
+
+def func_3(date_list):
+    if len(date_list) > 0:
+        print(list(set(date_list)))
+        for date_retry in list(set(date_list)):
             try:
-                locals()[table_name] = conn.query_df(
-                    f"select date,SUM(impressions) as impressions from AmazonReport.`{table_name}` group by date,profileId")
+                pre_func(AWS_CREDENTIALS, date_retry)
             except:
-                try:
-                    locals()[table_name] = conn.query_df(
-                        f"select date,SUM(salesOtherSku1d) as salesOtherSku1d from AmazonReport.`{table_name}` group by date,profileId")
-                except:
-                    try:
-                        locals()[table_name] = conn.query_df(
-                            f"select date,SUM(salesBrandHalo) as salesBrandHalo from AmazonReport.`{table_name}` group by date,profileId")
-                    except:
-                        locals()[table_name] = conn.query_df(
-                            f"select date,SUM(sales14d) as sales14d  from AmazonReport.`{table_name}` group by date,profileId")
-            locals()[table_name].columns = ['date', 'refer_']
-            locals()[table_name]['refer_'] = locals()[table_name]['refer_'].astype('float64')
-            todayMinus2 = (datetime.today().date() + timedelta(days=-2))
-            reback_21days = (todayMinus2 + timedelta(days=-23)).isoformat()
-            df = pd.date_range(start=reback_21days, end=todayMinus2.isoformat(), freq='1d')
-            df1 = pd.DataFrame(df, columns=['date_'])
-            merge_df = df1.merge(locals()[table_name], left_on=['date_'], right_on=['date'], how='left')
-            rel = merge_df.query("date_!=date or refer_<1 or refer_.isna()")['date_'].map(
-                lambda x: x.date().isoformat()).tolist()
-            print(rel, date_list)
-            if len(rel) > 0:
-                for date in rel:
-                    date_list.extend([date])
-                    conn.command(
-                        f"delete from AmazonReport.`{table_name}` where date='{date}' and profileId='{AWS_CREDENTIALS['profile_id']}'")
-        if len(date_list) > 0:
-            print(list(set(date_list)))
-            for date_retry in list(set(date_list)):
-                try:
-                    pre_func(AWS_CREDENTIALS, date_retry)
-                except:
-                    pre_func(AWS_CREDENTIALS, date_retry)
+                pre_func(AWS_CREDENTIALS, date_retry)
+
 
 if __name__ == '__main__':
     # AccountTask("3006125408623189").do({"record": "portfolios"})

start_sync_amz_RightNowRun.py

@@ -96,7 +96,7 @@ if __name__ == '__main__':
 
     # list_date = ["2024-03-23",]
     # print("11111111111111111111111111111")
-    list_date = [f'2024-07-{"0"+str(i) if len(str(i))==1 else i}' for i in range(22,23)]
+    list_date = [f'2024-07-{"0"+str(i) if len(str(i))==1 else i}' for i in range(11,13,1)]
     print(list_date)
     count = 1
     for date_ in list_date:

sync_amz_data/DataTransform/Data_ETL.py

@@ -2043,10 +2043,6 @@ class SD_ETL(SDClient, Common_ETLMethod):
         df_report = pd.json_normalize(list_report)
         df_report = self.type_trans(df_report, params['columns'], timeZone_, extra_columns=[])
 
-        print(df_report[params['columns']])
-        print(df_report[params['columns']].dtypes)
-        print(df_report[params['columns']].info())
-        # df_report[params['columns']].to_excel('temp000.xlsx',index=False)
         try:
             df_report['targetingText'] = df_report['targetingText'].fillna('-')
             df_report['targetingText'] = df_report['targetingText'].map(lambda x: str(x))