ammend sync

start_sync_amz.py

@@ -80,7 +80,7 @@ if __name__ == '__main__':
     timezone_ = Common_ETLMethod(**AWS_CREDENTIALS).timeZone()
     print(timezone_)
     sched = BlockingScheduler()
-    sched.add_job(lambda: amz_report(AWS_CREDENTIALS=AWS_CREDENTIALS),'cron',hour=17,minute=0,second=0,timezone=timezone_)#,params={"startDate":"2023-11-04","endDate":"2023-11-04","date":"20231104"}
+    sched.add_job(amz_report,'cron',hour=17,minute=0,second=0,timezone=timezone_, args=(AWS_CREDENTIALS, {}))#,params={"startDate":"2023-11-04","endDate":"2023-11-04","date":"20231104"}
     sched.start()
 
 

sync_amz_data/DataTransform/Data_ETL.py

@@ -122,6 +122,7 @@ class Common_ETLMethod(BaseClient):
         df_report.fillna(np.nan, inplace=True)
         # print(df_report[columns].info())
         # df_report.to_excel("df.xlsx")
+        params = {}
         return df_report
 
     def today_(self):
@@ -219,6 +220,7 @@ class SP_ETL(SPClient, Common_ETLMethod):
 
         conn.insert_df("AmazonReport.SP_spCampaigns_campaignV3", df_report[params['columns']])
         print("插入完成SP_spCampaigns_campaignV3")
+        params = {}
         return df_report[params['columns']]
 
     def reportV3_adGroup_spCampaignsETL(self, conn, params={}):
@@ -255,6 +257,7 @@ class SP_ETL(SPClient, Common_ETLMethod):
 
         conn.insert_df("AmazonReport.SP_spCampaigns_adGroupV3", df_report[params['columns']])
         print("插入完成SP_spCampaigns_adGroupV3")
+        params = {}
         return df_report[params['columns']]
 
     #
@@ -295,6 +298,7 @@ class SP_ETL(SPClient, Common_ETLMethod):
         conn.insert_df("AmazonReport.SP_spCampaigns_placementV3", df_report[params['columns']])
         print("插入完成SP_spCampaigns_placementV3")
 
+        params = {}
         return df_report[params['columns']]
 
     def reportV3_targeting_spTargetingETL(self, conn, params={}):
@@ -335,6 +339,7 @@ class SP_ETL(SPClient, Common_ETLMethod):
         conn.insert_df("AmazonReport.SP_spTargeting_targetingV3", df_report[params['columns']])
         print("插入完成SP_spTargeting_targetingV3")
 
+        params = {}
         return df_report[params['columns']]
 
     def reportV3_searchTerm_spSearchTermETL(self, conn, params={}):
@@ -375,6 +380,7 @@ class SP_ETL(SPClient, Common_ETLMethod):
         conn.insert_df("AmazonReport.SP_spSearchTerm_searchTermV3", df_report[params['columns']])
         print("插入完成SP_spSearchTerm_searchTermV3")
 
+        params = {}
         return df_report[params['columns']]
 
     def reportV3_advertiser_spAdvertisedProductETL(self, conn, params={}):
@@ -412,6 +418,7 @@ class SP_ETL(SPClient, Common_ETLMethod):
         conn.insert_df("AmazonReport.SP_spAdvertisedProduct_advertiserV3", df_report[params['columns']])
         print("插入完成SP_spAdvertisedProduct_advertiserV3")
 
+        params = {}
         return df_report[params['columns']]
 
     def reportV3_asin_spPurchasedProductETL(self, conn, params={}):
@@ -447,6 +454,7 @@ class SP_ETL(SPClient, Common_ETLMethod):
         conn.insert_df("AmazonReport.SP_spPurchasedProduct_asinV3", df_report[params['columns']])
         print("插入完成SP_spPurchasedProduct_asinV3")
 
+        params = {}
         return df_report[params['columns']]
 
 
@@ -529,6 +537,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         # conn.insert_df("AmazonReport.SB_sbPurchasedProduct_asinV3", df_report[params['columns']])
         # print("插入完成SB_sbPurchasedProduct_asinV3")
 
+        params = {}
         return df_report[params['columns']]
 
 
@@ -539,6 +548,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         if params.get("startDate") == None:
             params["startDate"] = (datetime(today.year, today.month, today.day) - timedelta(days=1)).strftime(
                 "%Y-%m-%d")
+        print(params)
         params['reportType'] = "sbPurchasedProduct"
         params['columns'] = [
             'campaignId', 'adGroupId', 'date', 'campaignBudgetCurrencyCode', 'campaignName', 'adGroupName',
@@ -560,6 +570,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         conn.insert_df("AmazonReport.SB_sbPurchasedProduct_asinV3", df_report[params['columns']])
         print("插入完成SB_sbPurchasedProduct_asinV3")
 
+        params = {}
         return df_report[params['columns']]
 
     def reportV2_campaignsRecord_ETL(self, conn, params={}):
@@ -592,6 +603,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         conn.insert_df("AmazonReport.SB_campaignsV2", df_report[metric])
         print("插入完成SB_campaignsV2")
 
+        params = {}
         return df_report[metric]
 
     def reportV2_campaignsVideo_ETL(self, conn, params={}):
@@ -625,6 +637,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         conn.insert_df("AmazonReport.SB_campaignsVideoV2", df_report[metric])
         print("插入完成SB_campaignsVideoV2")
         # print(df_report[metric].info())
+        params = {}
         return df_report[metric]
 
     def reportV2_placementRecord_ETL(self, conn, params={}):
@@ -654,6 +667,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         conn.insert_df("AmazonReport.SB_campaignsPlacementV2", df_report[metric])
         print("插入完成SB_campaignsPlacementV2")
         # print(df_report[metric].info())
+        params = {}
         return df_report[metric]
 
     def reportV2_placementVideo_ETL(self, conn, params={}):
@@ -685,6 +699,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         conn.insert_df("AmazonReport.SB_campaignsPlacementVideoV2", df_report[metric])
         print("插入完成SB_campaignsPlacementVideoV2")
 
+        params = {}
         return df_report[metric]
 
     def reportV2_adGroupsRecord_ETL(self, conn, params={}):
@@ -715,6 +730,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         conn.insert_df("AmazonReport.SB_adGroupsV2", df_report[metric])
         print("插入完成SB_adGroupsV2")
 
+        params = {}
         return df_report[metric]
 
     def reportV2_adGroupsVideo_ETL(self, conn, params={}):
@@ -745,6 +761,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
 
         conn.insert_df("AmazonReport.SB_adGroupsVideoV2", df_report[metric])
         print("插入完成SB_adGroupsVideoV2")
+        params = {}
         return df_report[metric]
 
     def reportV2_targetsRecord_ETL(self, conn, params={}):
@@ -775,6 +792,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         conn.insert_df("AmazonReport.SB_targetsV2", df_report[metric])
         print("插入完成SB_targetsV2")
 
+        params = {}
         return df_report[metric]
 
     def reportV2_targetsVideo_ETL(self, conn, params={}):
@@ -806,6 +824,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
 
         conn.insert_df("AmazonReport.SB_targetsVideoV2", df_report[metric])
         print("插入完成SB_targetsVideoV2")
+        params = {}
         return df_report[metric]
 
     def reportV2_keywordsRecord_ETL(self, conn, params={}):
@@ -836,6 +855,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
 
         conn.insert_df("AmazonReport.SB_keywordsV2", df_report[metric])
         print("插入完成SB_keywordsV2")
+        params = {}
         return df_report[metric]
 
     def reportV2_keywordsVideo_ETL(self, conn, params={}):
@@ -867,6 +887,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
 
         conn.insert_df("AmazonReport.SB_keywordsVideoV2", df_report[metric])
         print("插入完成SB_keywordsVideoV2")
+        params = {}
         return df_report[metric]
 
     def reportV2_searchtermsRecord_ETL(self, conn, params={}):
@@ -892,6 +913,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
 
         conn.insert_df("AmazonReport.SB_keywordsQueryV2", df_report[metric])
         print("插入完成SB_keywordsQueryV2")
+        params = {}
         return df_report[metric]
 
     def reportV2_searchtermsVideo_ETL(self, conn, params={}):
@@ -920,6 +942,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
 
         conn.insert_df("AmazonReport.SB_keywordsQueryVideoV2", df_report[metric])
         print("插入完成SB_keywordsQueryVideoV2")
+        params = {}
         return df_report[metric]
 
     def reportV2_adsRecord_ETL(self, conn, params={}):
@@ -948,6 +971,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
 
         conn.insert_df("AmazonReport.SB_adsV2", df_report[metric])
         print("插入完成SB_adsV2")
+        params = {}
         return df_report[metric]
 
     def reportV2_adsVideo_ETL(self, conn, params={}):
@@ -980,6 +1004,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
 
         conn.insert_df("AmazonReport.SB_adsVideoV2", df_report[metric])
         print("插入完成SB_adsVideoV2")
+        params = {}
         return df_report[metric]
 
 
@@ -1053,6 +1078,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         conn.insert_df("AmazonReport.SD_sdCampaigns_campaignV3", df_report[params['columns']])
         print("插入完成SD_sdCampaigns_campaignV3")
 
+        params = {}
         return df_report[params['columns']]
 
     def reportV3_campaignMT_sdCampaigns_ETL(self, conn, params={}):
@@ -1089,6 +1115,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         conn.insert_df("AmazonReport.SD_sdCampaigns_campaignMatchedTargetV3", df_report[params['columns']])
         print("插入完成SD_sdCampaigns_campaignMatchedTargetV3")
 
+        params = {}
         return df_report[params['columns']]
 
     def reportV3_adgroup_sdAdGroup_ETL(self, conn, params={}):
@@ -1126,6 +1153,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         conn.insert_df("AmazonReport.SD_sdAdGroup_adGroupV3", df_report[params['columns']])
         print("插入完成SD_sdAdGroup_adGroupV3")
 
+        params = {}
         return df_report[params['columns']]
 
     def reportV3_adgroupMT_sdAdGroup_ETL(self, conn, params={}):
@@ -1163,6 +1191,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         conn.insert_df("AmazonReport.SD_sdAdGroup_adGroupMatchedTargetV3", df_report[params['columns']])
         print("插入完成SD_sdAdGroup_adGroupMatchedTargetV3")
 
+        params = {}
         return df_report[params['columns']]
 
     def reportV3_targeting_sdTargeting_ETL(self, conn, params={}):
@@ -1204,6 +1233,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         conn.insert_df("AmazonReport.SD_targeting_sdTargetingV3", df_report[params['columns']])
         print("插入完成SD_targeting_sdTargetingV3")
 
+        params = {}
         return df_report[params['columns']]
 
     def reportV3_targetingMT_sdTargeting_ETL(self, conn, params={}):
@@ -1246,6 +1276,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         conn.insert_df("AmazonReport.SD_targeting_sdTargetingMatchedTargetV3", df_report[params['columns']])
         print("插入完成SD_targeting_sdTargetingMatchedTargetV3")
 
+        params = {}
         return df_report[params['columns']]
 
     def reportV3_advertiser_sdAdvertisedProduct_ETL(self, conn, params={}):
@@ -1285,6 +1316,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         conn.insert_df("AmazonReport.SD_advertiser_sdAdvertisedProductV3", df_report[params['columns']])
         print("插入完成SD_advertiser_sdAdvertisedProductV3")
 
+        params = {}
         return df_report[params['columns']]
 
     def reportV3_asin_sdPurchasedProduct_ETL(self, conn, params={}):
@@ -1316,6 +1348,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         conn.insert_df("AmazonReport.SD_asin_sdPurchasedProductV3", df_report[params['columns']])
         print("插入完成SD_asin_sdPurchasedProductV3")
 
+        params = {}
         return df_report[params['columns']]
 
     campaigns_metrics = [
@@ -1354,6 +1387,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
         conn.insert_df("AmazonReport.SD_campaignsV2", df_report[metric])
         print("插入完成SD_campaignsV2—")
+        params = {}
         return df_report[metric]
 
     def reportV2_campaignsRecord_t3_ETL(self, conn, params={}):
@@ -1376,6 +1410,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
         conn.insert_df("AmazonReport.SD_campaignsV2", df_report[metric])
         print("插入完成SD_campaignsV2")
+        params = {}
         return df_report[metric]
 
     adGroups_metrics = [
@@ -1411,6 +1446,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
         conn.insert_df("AmazonReport.SD_adGroupsV2", df_report[metric])
         print("插入完成SD_adGroupsV2")
+        params = {}
         return df_report[metric]
 
     def reportV2_adGroupsRecord_t3_ETL(self, conn, params={}):
@@ -1431,6 +1467,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
         conn.insert_df("AmazonReport.SD_adGroupsV2", df_report[metric])
         print("插入完成SD_adGroupsV2")
+        params = {}
         return df_report[metric]
 
     productAds_metrics = [
@@ -1466,6 +1503,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
         conn.insert_df("AmazonReport.SD_adsV2", df_report[metric])
         print("插入完成SD_adsV2")
+        params = {}
         return df_report[metric]
 
     def reportV2_productAds_t3_ETL(self, conn, params={}):
@@ -1487,6 +1525,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
         conn.insert_df("AmazonReport.SD_adsV2", df_report[metric])
         print("插入完成SD_adsV2")
+        params = {}
         return df_report[metric]
 
     targets_metrics = [
@@ -1524,6 +1563,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         # print(df_report[metric].info())
         conn.insert_df("AmazonReport.SD_targetsV2", df_report[metric])
         print("插入完成SD_targetsV2")
+        params = {}
         return df_report[metric]
 
     def reportV2_targets_t3_ETL(self, conn, params={}):
@@ -1547,6 +1587,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
         conn.insert_df("AmazonReport.SD_targetsV2", df_report[metric])
         print("插入完成SD_targetsV2")
+        params = {}
         return df_report[metric]
 
     asins_metrics = [
@@ -1581,6 +1622,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         # print(df_report.info())
         conn.insert_df("AmazonReport.SD_asinsV2", df_report[metric])
         print("插入完成SD_asinsV2")
+        params = {}
         return df_report[metric]
 
     def reportV2_asins_t3_ETL(self, conn, params={}):
@@ -1603,6 +1645,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
         conn.insert_df("AmazonReport.SD_asinsV2", df_report[metric])
         print("插入完成SD_asinsV2")
+        params = {}
         return df_report[metric]
 
     campaigns_MT_metrics = [
@@ -1643,6 +1686,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
         conn.insert_df("AmazonReport.SD_campaignsMatchedTargetV2", df_report[metric])
         print("插入完成SD_campaignsMatchedTargetV2")
+        params = {}
         return df_report[metric]
 
     def reportV2_campaign_matchedTarget_t3_ETL(self, conn, params={}):
@@ -1665,6 +1709,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
         conn.insert_df("AmazonReport.SD_campaignsMatchedTargetV2", df_report[metric])
         print("插入完成SD_campaignsMatchedTargetV2")
+        params = {}
         return df_report[metric]
 
     adGroups_MT_metrics = [
@@ -1702,6 +1747,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
         conn.insert_df("AmazonReport.SD_adGroupsMatchedTargetV2", df_report[metric])
         print("插入完成SD_adGroupsMatchedTargetV2")
+        params = {}
         return df_report[metric]
 
     def reportV2_adGroups_matchedTarget_t3_ETL(self, conn, params={}):
@@ -1723,6 +1769,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         df_report = self.type_trans(df_report, metric, timeZone_, extra_columns=['matchedTarget', 'tactic'])
         conn.insert_df("AmazonReport.SD_adGroupsMatchedTargetV2", df_report[metric])
         print("插入完成SD_adGroupsMatchedTargetV2")
+        params = {}
         return df_report[metric]
 
     targets_MT_metrics = [
@@ -1758,6 +1805,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
         conn.insert_df("AmazonReport.SD_targetsMatchedTargetV2", df_report[metric])
         print("插入完成SD_targetsMatchedTargetV2")
+        params = {}
         return df_report[metric]
 
     def reportV2_targets_matchedTarget_t3_ETL(self, conn, params={}):
@@ -1778,6 +1826,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
         conn.insert_df("AmazonReport.SD_targetsMatchedTargetV2", df_report[metric])
         print("插入完成SD_targetsMatchedTargetV2")
+        params = {}
         return df_report[metric]
 
 
@@ -1791,7 +1840,7 @@ if __name__ == '__main__':
     conn = Common_ETLMethod(**AWS_CREDENTIALS).clickhouse_connect()
 
     # SD
-    ac_etl = SB_ETL(**AWS_CREDENTIALS)
+    ac_etl = SP_ETL(**AWS_CREDENTIALS)
     # ls = ac_etl.reportV2_campaignsRecord_t2_ETL(conn)
     ls = ac_etl.targets_ETL()
     print(ls)