ホーム エクスプローラ ヘルプ
登録 サインイン
ASJ_ADS
/
sync_amz_data
2
0
フォーク 0
ファイル 課題 0 プルリクエスト 0 Wiki
ソースを参照

ammend sync

huangyifan 1 年間 前
親
5f81d55220
コミット
5e48591ad7
2 ファイル変更、51 行追加、2 行削除
分割表示 差分情報を表示
  1. 1 1
      start_sync_amz.py
  2. 50 1
      sync_amz_data/DataTransform/Data_ETL.py

+ 1 - 1
start_sync_amz.py
ファイルの表示 

@@ -80,7 +80,7 @@ if __name__ == '__main__':
 
     timezone_ = Common_ETLMethod(**AWS_CREDENTIALS).timeZone()
 
     print(timezone_)
 
     sched = BlockingScheduler()
 
-    sched.add_job(lambda: amz_report(AWS_CREDENTIALS=AWS_CREDENTIALS),'cron',hour=17,minute=0,second=0,timezone=timezone_)#,params={"startDate":"2023-11-04","endDate":"2023-11-04","date":"20231104"}
 
+    sched.add_job(amz_report,'cron',hour=17,minute=0,second=0,timezone=timezone_, args=(AWS_CREDENTIALS, {}))#,params={"startDate":"2023-11-04","endDate":"2023-11-04","date":"20231104"}
 
     sched.start()

+ 50 - 1
sync_amz_data/DataTransform/Data_ETL.py
ファイルの表示 

@@ -122,6 +122,7 @@ class Common_ETLMethod(BaseClient):
 
         df_report.fillna(np.nan, inplace=True)
 
         # print(df_report[columns].info())
 
         # df_report.to_excel("df.xlsx")
 
+        params = {}
 
         return df_report
 
 
     def today_(self):
 
@@ -219,6 +220,7 @@ class SP_ETL(SPClient, Common_ETLMethod):
 
 
         conn.insert_df("AmazonReport.SP_spCampaigns_campaignV3", df_report[params['columns']])
 
         print("插入完成SP_spCampaigns_campaignV3")
 
+        params = {}
 
         return df_report[params['columns']]
 
 
     def reportV3_adGroup_spCampaignsETL(self, conn, params={}):
 
@@ -255,6 +257,7 @@ class SP_ETL(SPClient, Common_ETLMethod):
 
 
         conn.insert_df("AmazonReport.SP_spCampaigns_adGroupV3", df_report[params['columns']])
 
         print("插入完成SP_spCampaigns_adGroupV3")
 
+        params = {}
 
         return df_report[params['columns']]
 
 
     #
 
@@ -295,6 +298,7 @@ class SP_ETL(SPClient, Common_ETLMethod):
 
         conn.insert_df("AmazonReport.SP_spCampaigns_placementV3", df_report[params['columns']])
 
         print("插入完成SP_spCampaigns_placementV3")
 
 
+        params = {}
 
         return df_report[params['columns']]
 
 
     def reportV3_targeting_spTargetingETL(self, conn, params={}):
 
@@ -335,6 +339,7 @@ class SP_ETL(SPClient, Common_ETLMethod):
 
         conn.insert_df("AmazonReport.SP_spTargeting_targetingV3", df_report[params['columns']])
 
         print("插入完成SP_spTargeting_targetingV3")
 
 
+        params = {}
 
         return df_report[params['columns']]
 
 
     def reportV3_searchTerm_spSearchTermETL(self, conn, params={}):
 
@@ -375,6 +380,7 @@ class SP_ETL(SPClient, Common_ETLMethod):
 
         conn.insert_df("AmazonReport.SP_spSearchTerm_searchTermV3", df_report[params['columns']])
 
         print("插入完成SP_spSearchTerm_searchTermV3")
 
 
+        params = {}
 
         return df_report[params['columns']]
 
 
     def reportV3_advertiser_spAdvertisedProductETL(self, conn, params={}):
 
@@ -412,6 +418,7 @@ class SP_ETL(SPClient, Common_ETLMethod):
 
         conn.insert_df("AmazonReport.SP_spAdvertisedProduct_advertiserV3", df_report[params['columns']])
 
         print("插入完成SP_spAdvertisedProduct_advertiserV3")
 
 
+        params = {}
 
         return df_report[params['columns']]
 
 
     def reportV3_asin_spPurchasedProductETL(self, conn, params={}):
 
@@ -447,6 +454,7 @@ class SP_ETL(SPClient, Common_ETLMethod):
 
         conn.insert_df("AmazonReport.SP_spPurchasedProduct_asinV3", df_report[params['columns']])
 
         print("插入完成SP_spPurchasedProduct_asinV3")
 
 
+        params = {}
 
         return df_report[params['columns']]
 
 
 
@@ -529,6 +537,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
 
         # conn.insert_df("AmazonReport.SB_sbPurchasedProduct_asinV3", df_report[params['columns']])
 
         # print("插入完成SB_sbPurchasedProduct_asinV3")
 
 
+        params = {}
 
         return df_report[params['columns']]
 
 
 
@@ -539,6 +548,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
 
         if params.get("startDate") == None:
 
             params["startDate"] = (datetime(today.year, today.month, today.day) - timedelta(days=1)).strftime(
 
                 "%Y-%m-%d")
 
+        print(params)
 
         params['reportType'] = "sbPurchasedProduct"
 
         params['columns'] = [
 
             'campaignId', 'adGroupId', 'date', 'campaignBudgetCurrencyCode', 'campaignName', 'adGroupName',
 
@@ -560,6 +570,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
 
         conn.insert_df("AmazonReport.SB_sbPurchasedProduct_asinV3", df_report[params['columns']])
 
         print("插入完成SB_sbPurchasedProduct_asinV3")
 
 
+        params = {}
 
         return df_report[params['columns']]
 
 
     def reportV2_campaignsRecord_ETL(self, conn, params={}):
 
@@ -592,6 +603,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
 
         conn.insert_df("AmazonReport.SB_campaignsV2", df_report[metric])
 
         print("插入完成SB_campaignsV2")
 
 
+        params = {}
 
         return df_report[metric]
 
 
     def reportV2_campaignsVideo_ETL(self, conn, params={}):
 
@@ -625,6 +637,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
 
         conn.insert_df("AmazonReport.SB_campaignsVideoV2", df_report[metric])
 
         print("插入完成SB_campaignsVideoV2")
 
         # print(df_report[metric].info())
 
+        params = {}
 
         return df_report[metric]
 
 
     def reportV2_placementRecord_ETL(self, conn, params={}):
 
@@ -654,6 +667,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
 
         conn.insert_df("AmazonReport.SB_campaignsPlacementV2", df_report[metric])
 
         print("插入完成SB_campaignsPlacementV2")
 
         # print(df_report[metric].info())
 
+        params = {}
 
         return df_report[metric]
 
 
     def reportV2_placementVideo_ETL(self, conn, params={}):
 
@@ -685,6 +699,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
 
         conn.insert_df("AmazonReport.SB_campaignsPlacementVideoV2", df_report[metric])
 
         print("插入完成SB_campaignsPlacementVideoV2")
 
 
+        params = {}
 
         return df_report[metric]
 
 
     def reportV2_adGroupsRecord_ETL(self, conn, params={}):
 
@@ -715,6 +730,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
 
         conn.insert_df("AmazonReport.SB_adGroupsV2", df_report[metric])
 
         print("插入完成SB_adGroupsV2")
 
 
+        params = {}
 
         return df_report[metric]
 
 
     def reportV2_adGroupsVideo_ETL(self, conn, params={}):
 
@@ -745,6 +761,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
 
 
         conn.insert_df("AmazonReport.SB_adGroupsVideoV2", df_report[metric])
 
         print("插入完成SB_adGroupsVideoV2")
 
+        params = {}
 
         return df_report[metric]
 
 
     def reportV2_targetsRecord_ETL(self, conn, params={}):
 
@@ -775,6 +792,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
 
         conn.insert_df("AmazonReport.SB_targetsV2", df_report[metric])
 
         print("插入完成SB_targetsV2")
 
 
+        params = {}
 
         return df_report[metric]
 
 
     def reportV2_targetsVideo_ETL(self, conn, params={}):
 
@@ -806,6 +824,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
 
 
         conn.insert_df("AmazonReport.SB_targetsVideoV2", df_report[metric])
 
         print("插入完成SB_targetsVideoV2")
 
+        params = {}
 
         return df_report[metric]
 
 
     def reportV2_keywordsRecord_ETL(self, conn, params={}):
 
@@ -836,6 +855,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
 
 
         conn.insert_df("AmazonReport.SB_keywordsV2", df_report[metric])
 
         print("插入完成SB_keywordsV2")
 
+        params = {}
 
         return df_report[metric]
 
 
     def reportV2_keywordsVideo_ETL(self, conn, params={}):
 
@@ -867,6 +887,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
 
 
         conn.insert_df("AmazonReport.SB_keywordsVideoV2", df_report[metric])
 
         print("插入完成SB_keywordsVideoV2")
 
+        params = {}
 
         return df_report[metric]
 
 
     def reportV2_searchtermsRecord_ETL(self, conn, params={}):
 
@@ -892,6 +913,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
 
 
         conn.insert_df("AmazonReport.SB_keywordsQueryV2", df_report[metric])
 
         print("插入完成SB_keywordsQueryV2")
 
+        params = {}
 
         return df_report[metric]
 
 
     def reportV2_searchtermsVideo_ETL(self, conn, params={}):
 
@@ -920,6 +942,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
 
 
         conn.insert_df("AmazonReport.SB_keywordsQueryVideoV2", df_report[metric])
 
         print("插入完成SB_keywordsQueryVideoV2")
 
+        params = {}
 
         return df_report[metric]
 
 
     def reportV2_adsRecord_ETL(self, conn, params={}):
 
@@ -948,6 +971,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
 
 
         conn.insert_df("AmazonReport.SB_adsV2", df_report[metric])
 
         print("插入完成SB_adsV2")
 
+        params = {}
 
         return df_report[metric]
 
 
     def reportV2_adsVideo_ETL(self, conn, params={}):
 
@@ -980,6 +1004,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
 
 
         conn.insert_df("AmazonReport.SB_adsVideoV2", df_report[metric])
 
         print("插入完成SB_adsVideoV2")
 
+        params = {}
 
         return df_report[metric]
 
 
 
@@ -1053,6 +1078,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
         conn.insert_df("AmazonReport.SD_sdCampaigns_campaignV3", df_report[params['columns']])
 
         print("插入完成SD_sdCampaigns_campaignV3")
 
 
+        params = {}
 
         return df_report[params['columns']]
 
 
     def reportV3_campaignMT_sdCampaigns_ETL(self, conn, params={}):
 
@@ -1089,6 +1115,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
         conn.insert_df("AmazonReport.SD_sdCampaigns_campaignMatchedTargetV3", df_report[params['columns']])
 
         print("插入完成SD_sdCampaigns_campaignMatchedTargetV3")
 
 
+        params = {}
 
         return df_report[params['columns']]
 
 
     def reportV3_adgroup_sdAdGroup_ETL(self, conn, params={}):
 
@@ -1126,6 +1153,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
         conn.insert_df("AmazonReport.SD_sdAdGroup_adGroupV3", df_report[params['columns']])
 
         print("插入完成SD_sdAdGroup_adGroupV3")
 
 
+        params = {}
 
         return df_report[params['columns']]
 
 
     def reportV3_adgroupMT_sdAdGroup_ETL(self, conn, params={}):
 
@@ -1163,6 +1191,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
         conn.insert_df("AmazonReport.SD_sdAdGroup_adGroupMatchedTargetV3", df_report[params['columns']])
 
         print("插入完成SD_sdAdGroup_adGroupMatchedTargetV3")
 
 
+        params = {}
 
         return df_report[params['columns']]
 
 
     def reportV3_targeting_sdTargeting_ETL(self, conn, params={}):
 
@@ -1204,6 +1233,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
         conn.insert_df("AmazonReport.SD_targeting_sdTargetingV3", df_report[params['columns']])
 
         print("插入完成SD_targeting_sdTargetingV3")
 
 
+        params = {}
 
         return df_report[params['columns']]
 
 
     def reportV3_targetingMT_sdTargeting_ETL(self, conn, params={}):
 
@@ -1246,6 +1276,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
         conn.insert_df("AmazonReport.SD_targeting_sdTargetingMatchedTargetV3", df_report[params['columns']])
 
         print("插入完成SD_targeting_sdTargetingMatchedTargetV3")
 
 
+        params = {}
 
         return df_report[params['columns']]
 
 
     def reportV3_advertiser_sdAdvertisedProduct_ETL(self, conn, params={}):
 
@@ -1285,6 +1316,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
         conn.insert_df("AmazonReport.SD_advertiser_sdAdvertisedProductV3", df_report[params['columns']])
 
         print("插入完成SD_advertiser_sdAdvertisedProductV3")
 
 
+        params = {}
 
         return df_report[params['columns']]
 
 
     def reportV3_asin_sdPurchasedProduct_ETL(self, conn, params={}):
 
@@ -1316,6 +1348,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
         conn.insert_df("AmazonReport.SD_asin_sdPurchasedProductV3", df_report[params['columns']])
 
         print("插入完成SD_asin_sdPurchasedProductV3")
 
 
+        params = {}
 
         return df_report[params['columns']]
 
 
     campaigns_metrics = [
 
@@ -1354,6 +1387,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
 
         conn.insert_df("AmazonReport.SD_campaignsV2", df_report[metric])
 
         print("插入完成SD_campaignsV2—")
 
+        params = {}
 
         return df_report[metric]
 
 
     def reportV2_campaignsRecord_t3_ETL(self, conn, params={}):
 
@@ -1376,6 +1410,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
 
         conn.insert_df("AmazonReport.SD_campaignsV2", df_report[metric])
 
         print("插入完成SD_campaignsV2")
 
+        params = {}
 
         return df_report[metric]
 
 
     adGroups_metrics = [
 
@@ -1411,6 +1446,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
 
         conn.insert_df("AmazonReport.SD_adGroupsV2", df_report[metric])
 
         print("插入完成SD_adGroupsV2")
 
+        params = {}
 
         return df_report[metric]
 
 
     def reportV2_adGroupsRecord_t3_ETL(self, conn, params={}):
 
@@ -1431,6 +1467,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
 
         conn.insert_df("AmazonReport.SD_adGroupsV2", df_report[metric])
 
         print("插入完成SD_adGroupsV2")
 
+        params = {}
 
         return df_report[metric]
 
 
     productAds_metrics = [
 
@@ -1466,6 +1503,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
 
         conn.insert_df("AmazonReport.SD_adsV2", df_report[metric])
 
         print("插入完成SD_adsV2")
 
+        params = {}
 
         return df_report[metric]
 
 
     def reportV2_productAds_t3_ETL(self, conn, params={}):
 
@@ -1487,6 +1525,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
 
         conn.insert_df("AmazonReport.SD_adsV2", df_report[metric])
 
         print("插入完成SD_adsV2")
 
+        params = {}
 
         return df_report[metric]
 
 
     targets_metrics = [
 
@@ -1524,6 +1563,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
         # print(df_report[metric].info())
 
         conn.insert_df("AmazonReport.SD_targetsV2", df_report[metric])
 
         print("插入完成SD_targetsV2")
 
+        params = {}
 
         return df_report[metric]
 
 
     def reportV2_targets_t3_ETL(self, conn, params={}):
 
@@ -1547,6 +1587,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
 
         conn.insert_df("AmazonReport.SD_targetsV2", df_report[metric])
 
         print("插入完成SD_targetsV2")
 
+        params = {}
 
         return df_report[metric]
 
 
     asins_metrics = [
 
@@ -1581,6 +1622,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
         # print(df_report.info())
 
         conn.insert_df("AmazonReport.SD_asinsV2", df_report[metric])
 
         print("插入完成SD_asinsV2")
 
+        params = {}
 
         return df_report[metric]
 
 
     def reportV2_asins_t3_ETL(self, conn, params={}):
 
@@ -1603,6 +1645,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
 
         conn.insert_df("AmazonReport.SD_asinsV2", df_report[metric])
 
         print("插入完成SD_asinsV2")
 
+        params = {}
 
         return df_report[metric]
 
 
     campaigns_MT_metrics = [
 
@@ -1643,6 +1686,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
 
         conn.insert_df("AmazonReport.SD_campaignsMatchedTargetV2", df_report[metric])
 
         print("插入完成SD_campaignsMatchedTargetV2")
 
+        params = {}
 
         return df_report[metric]
 
 
     def reportV2_campaign_matchedTarget_t3_ETL(self, conn, params={}):
 
@@ -1665,6 +1709,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
 
         conn.insert_df("AmazonReport.SD_campaignsMatchedTargetV2", df_report[metric])
 
         print("插入完成SD_campaignsMatchedTargetV2")
 
+        params = {}
 
         return df_report[metric]
 
 
     adGroups_MT_metrics = [
 
@@ -1702,6 +1747,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
 
         conn.insert_df("AmazonReport.SD_adGroupsMatchedTargetV2", df_report[metric])
 
         print("插入完成SD_adGroupsMatchedTargetV2")
 
+        params = {}
 
         return df_report[metric]
 
 
     def reportV2_adGroups_matchedTarget_t3_ETL(self, conn, params={}):
 
@@ -1723,6 +1769,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
         df_report = self.type_trans(df_report, metric, timeZone_, extra_columns=['matchedTarget', 'tactic'])
 
         conn.insert_df("AmazonReport.SD_adGroupsMatchedTargetV2", df_report[metric])
 
         print("插入完成SD_adGroupsMatchedTargetV2")
 
+        params = {}
 
         return df_report[metric]
 
 
     targets_MT_metrics = [
 
@@ -1758,6 +1805,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
 
         conn.insert_df("AmazonReport.SD_targetsMatchedTargetV2", df_report[metric])
 
         print("插入完成SD_targetsMatchedTargetV2")
 
+        params = {}
 
         return df_report[metric]
 
 
     def reportV2_targets_matchedTarget_t3_ETL(self, conn, params={}):
 
@@ -1778,6 +1826,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
 
         conn.insert_df("AmazonReport.SD_targetsMatchedTargetV2", df_report[metric])
 
         print("插入完成SD_targetsMatchedTargetV2")
 
+        params = {}
 
         return df_report[metric]
 
 
 
@@ -1791,7 +1840,7 @@ if __name__ == '__main__':
 
     conn = Common_ETLMethod(**AWS_CREDENTIALS).clickhouse_connect()
 
 
     # SD
 
-    ac_etl = SB_ETL(**AWS_CREDENTIALS)
 
+    ac_etl = SP_ETL(**AWS_CREDENTIALS)
 
     # ls = ac_etl.reportV2_campaignsRecord_t2_ETL(conn)
 
     ls = ac_etl.targets_ETL()
 
     print(ls)