Merge branch 'yifan' of ASJ_ADS/sync_amz_data into master

start_sync_amz_RightNowRun.py

@@ -96,7 +96,7 @@ if __name__ == '__main__':
 
     # list_date = ["2024-03-23",]
     # print("11111111111111111111111111111")
-    list_date = [f'2024-07-{"0"+str(i) if len(str(i))==1 else i}' for i in range(4,20)]
+    list_date = [f'2024-07-{"0"+str(i) if len(str(i))==1 else i}' for i in range(22,23)]
     print(list_date)
     count = 1
     for date_ in list_date:

sync_amz_data/DataTransform/Data_ETL.py

@@ -37,7 +37,17 @@ class Common_ETLMethod(BaseClient):
             except Exception as e:
                 print(e, '重试2')
                 time.sleep(15)
-                conn = self.conn_detail()
+                try:
+                    conn = self.conn_detail()
+                except Exception as e:
+                    print(e, '重试3')
+                    try:
+                        time.sleep(10)
+                        conn = self.conn_detail()
+                    except Exception as e:
+                        print(e, '重试4')
+                        time.sleep(15)
+                        conn = self.conn_detail()
         return conn
 
     def conn_detail(self):
@@ -158,6 +168,9 @@ class Common_ETLMethod(BaseClient):
             df_report["searchTermImpressionShare"] = df_report["searchTermImpressionShare"].fillna(0.0)
         if "impressionsFrequencyAverage" in df_report.columns:
             df_report["impressionsFrequencyAverage"] = df_report["impressionsFrequencyAverage"].fillna(0.0)
+        # if 'newToBrandSalesClicks' in df_report.columns:
+        #     df_report['newToBrandSalesClicks'] = df_report['newToBrandSalesClicks'].map(lambda x: round(x, 0))
+        #     df_report['newToBrandSalesClicks'] = df_report['newToBrandSalesClicks'].astype('int32')
         df_report.fillna(np.nan, inplace=True)
         # print(df_report[columns].info())
         # df_report.to_excel("df.xlsx")
@@ -305,7 +318,6 @@ class SP_ETL(SPClient, Common_ETLMethod):
             except Exception as e:
                 print('插入出错，重试失败。错误原因: ',e)
                 return ''
-            return ''
         time.sleep(1.05)
         print("插入完成SP_spCampaigns_campaignV3")
 
@@ -358,7 +370,6 @@ class SP_ETL(SPClient, Common_ETLMethod):
             except Exception as e:
                 print('插入出错，重试失败。错误原因: ',e)
                 return ''
-            return ''
         time.sleep(1.05)
         print("插入完成SP_spCampaigns_adGroupV3")
 
@@ -413,7 +424,6 @@ class SP_ETL(SPClient, Common_ETLMethod):
             except Exception as e:
                 print('插入出错，重试失败。错误原因: ',e)
                 return ''
-            return ''
         time.sleep(1.05)
         print("插入完成SP_spCampaigns_placementV3")
 
@@ -470,7 +480,6 @@ class SP_ETL(SPClient, Common_ETLMethod):
             except Exception as e:
                 print('插入出错，重试失败。错误原因: ',e)
                 return ''
-            return ''
         time.sleep(1.05)
         print("插入完成SP_spTargeting_targetingV3")
 
@@ -527,7 +536,6 @@ class SP_ETL(SPClient, Common_ETLMethod):
             except Exception as e:
                 print('插入出错，重试失败。错误原因: ',e)
                 return ''
-            return ''
         time.sleep(1.05)
         print("插入完成SP_spSearchTerm_searchTermV3")
 
@@ -570,7 +578,6 @@ class SP_ETL(SPClient, Common_ETLMethod):
         # print(list_report)
         df_report = pd.json_normalize(list_report)
         df_report = self.type_trans(df_report, params['columns'], timeZone_, extra_columns=[])
-        # TODO
         df_report[df_report.select_dtypes(["object"]).columns] = df_report[df_report.select_dtypes(["object"]).columns].fillna('')
         df_report[df_report.select_dtypes(["string"]).columns] = df_report[df_report.select_dtypes(["string"]).columns].fillna('')
 
@@ -642,7 +649,6 @@ class SP_ETL(SPClient, Common_ETLMethod):
                 except Exception as e:
                     print('插入出错，重试失败。错误原因: ',e)
                     return ''
-                return ''
             time.sleep(1.05)
             print("插入完成SP_spPurchasedProduct_asinV3")
 
@@ -745,7 +751,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
             except Exception as e:
                 print('插入出错，重试失败。错误原因: ',e)
                 return ''
-            return ''
+
         time.sleep(1.05)
         print("插入完成SB_sbCampaigns_campaignV3")
         df_rel =  df_report[params['columns']]
@@ -793,7 +799,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
             except Exception as e:
                 print('插入出错，重试失败。错误原因: ',e)
                 return ''
-            return ''
+
         time.sleep(1.05)
         print("插入完成SB_sbAdGroup_adGroupV3")
         df_rel =  df_report[params['columns']]
@@ -842,7 +848,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
             except Exception as e:
                 print('插入出错，重试失败。错误原因: ',e)
                 return ''
-            return ''
+
         time.sleep(1.05)
         print("插入完成SB_sbCampaigns_placementV3")
         df_rel =  df_report[params['columns']]
@@ -893,7 +899,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
             except Exception as e:
                 print('插入出错，重试失败。错误原因: ',e)
                 return ''
-            return ''
+
         time.sleep(1.05)
         print("插入完成SB_sbTargeting_targetingV3")
         df_rel =  df_report[params['columns']]
@@ -939,7 +945,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
             except Exception as e:
                 print('插入出错，重试失败。错误原因: ',e)
                 return ''
-            return ''
+
         time.sleep(1.05)
         print("插入完成SB_sbSearchTerm_searchTermV3")
         df_rel =  df_report[params['columns']]
@@ -992,7 +998,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
             except Exception as e:
                 print('插入出错，重试失败。错误原因: ',e)
                 return ''
-            return ''
+
         time.sleep(1.05)
         print("插入完成SB_sbAds_adsV3")
 
@@ -1040,7 +1046,6 @@ class SB_ETL(SBClient, Common_ETLMethod):
             except Exception as e:
                 print('插入出错，重试失败。错误原因: ',e)
                 return ''
-            return ''
         time.sleep(1.05)
         print("插入完成SB_sbPurchasedProduct_asinV3")
 
@@ -1091,7 +1096,6 @@ class SB_ETL(SBClient, Common_ETLMethod):
             except Exception as e:
                 print('插入出错，重试失败。错误原因: ',e)
                 return ''
-            return ''
         time.sleep(1.05)
         print("插入完成SB_campaignsV2")
 
@@ -1144,7 +1148,6 @@ class SB_ETL(SBClient, Common_ETLMethod):
             except Exception as e:
                 print('插入出错，重试失败。错误原因: ',e)
                 return ''
-            return ''
         time.sleep(1.05)
         print("插入完成SB_campaignsVideoV2")
         # print(df_report[metric].info())
@@ -1193,7 +1196,6 @@ class SB_ETL(SBClient, Common_ETLMethod):
             except Exception as e:
                 print('插入出错，重试失败。错误原因: ',e)
                 return ''
-            return ''
         time.sleep(1.05)
         print("插入完成SB_campaignsPlacementV2")
         # print(df_report[metric].info())
@@ -1244,7 +1246,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
             except Exception as e:
                 print('插入出错，重试失败。错误原因: ',e)
                 return ''
-            return ''
+
         time.sleep(1.05)
         print("插入完成SB_campaignsPlacementVideoV2")
 
@@ -1294,7 +1296,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
             except Exception as e:
                 print('插入出错，重试失败。错误原因: ',e)
                 return ''
-            return ''
+
         time.sleep(1.05)
         print("插入完成SB_adGroupsV2")
 
@@ -1344,7 +1346,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
             except Exception as e:
                 print('插入出错，重试失败。错误原因: ',e)
                 return ''
-            return ''
+
         time.sleep(1.05)
         print("插入完成SB_adGroupsVideoV2")
 
@@ -1393,7 +1395,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
             except Exception as e:
                 print('插入出错，重试失败。错误原因: ',e)
                 return ''
-            return ''
+
         time.sleep(1.05)
         print("插入完成SB_targetsV2")
 
@@ -1445,7 +1447,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
             except Exception as e:
                 print('插入出错，重试失败。错误原因: ',e)
                 return ''
-            return ''
+
         time.sleep(1.05)
         print("插入完成SB_targetsVideoV2")
 
@@ -1495,7 +1497,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
             except Exception as e:
                 print('插入出错，重试失败。错误原因: ',e)
                 return ''
-            return ''
+
         time.sleep(1.05)
         print("插入完成SB_keywordsV2")
 
@@ -1546,7 +1548,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
             except Exception as e:
                 print('插入出错，重试失败。错误原因: ',e)
                 return ''
-            return ''
+
         time.sleep(1.05)
         print("插入完成SB_keywordsVideoV2")
 
@@ -1591,7 +1593,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
             except Exception as e:
                 print('插入出错，重试失败。错误原因: ',e)
                 return ''
-            return ''
+
         time.sleep(1.05)
         print("插入完成SB_keywordsQueryV2")
 
@@ -1639,7 +1641,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
             except Exception as e:
                 print('插入出错，重试失败。错误原因: ',e)
                 return ''
-            return ''
+
         time.sleep(1.05)
         print("插入完成SB_keywordsQueryVideoV2")
 
@@ -1687,7 +1689,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
             except Exception as e:
                 print('插入出错，重试失败。错误原因: ',e)
                 return ''
-            return ''
+
         time.sleep(1.05)
         print("插入完成SB_adsV2")
 
@@ -1739,7 +1741,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
             except Exception as e:
                 print('插入出错，重试失败。错误原因: ',e)
                 return ''
-            return ''
+
         time.sleep(1.05)
         print("插入完成SB_adsVideoV2")
 
@@ -1829,7 +1831,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
             except Exception as e:
                 print('插入出错，重试失败。错误原因: ',e)
                 return ''
-            return ''
+
         time.sleep(1.05)
         print("插入完成SD_sdCampaigns_campaignV3")
 
@@ -1882,7 +1884,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
             except Exception as e:
                 print('插入出错，重试失败。错误原因: ',e)
                 return ''
-            return ''
+
         time.sleep(1.05)
         print("插入完成SD_sdCampaigns_campaignMatchedTargetV3")
 
@@ -1936,7 +1938,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
             except Exception as e:
                 print('插入出错，重试失败。错误原因: ',e)
                 return ''
-            return ''
+
         time.sleep(1.05)
         print("插入完成SD_sdAdGroup_adGroupV3")
 
@@ -1990,7 +1992,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
             except Exception as e:
                 print('插入出错，重试失败。错误原因: ',e)
                 return ''
-            return ''
+
         time.sleep(1.05)
         print("插入完成SD_sdAdGroup_adGroupMatchedTargetV3")
 
@@ -2007,8 +2009,12 @@ class SD_ETL(SDClient, Common_ETLMethod):
         if len(conn.query_df(
                 f"select * from AmazonReport.SD_targeting_sdTargetingV3 where date='{params['startDate']}' limit 1")) > 0:
             logging.info("数据已存在...")
+            print("999999999999999999999")
             time.sleep(8.5)
             return 'Pass'
+        a=conn.query_df(
+                f"select * from AmazonReport.SD_targeting_sdTargetingV3 limit 1")
+
         params['reportType'] = "sdTargeting"
         params['columns'] = ['campaignName', 'campaignId','adGroupName','adGroupId', 'targetingText','targetingId','impressions','clicks', 'cost',
             'adKeywordStatus', 'addToCart', 'addToCartClicks', 'addToCartRate', 'addToCartViews',
@@ -2035,27 +2041,31 @@ class SD_ETL(SDClient, Common_ETLMethod):
                                          endDate=params['endDate'], reportType=params['reportType'])
         # print(list_report)
         df_report = pd.json_normalize(list_report)
-        # print("=====",df_report,sep='\n')
-        # print(df_report.to_excel('temp.xlsx',index=False))
-        # print(df_report.dtypes)
         df_report = self.type_trans(df_report, params['columns'], timeZone_, extra_columns=[])
 
-        # print(df_report)
+        print(df_report[params['columns']])
+        print(df_report[params['columns']].dtypes)
+        print(df_report[params['columns']].info())
+        # df_report[params['columns']].to_excel('temp000.xlsx',index=False)
         try:
+            df_report['targetingText'] = df_report['targetingText'].fillna('-')
+            df_report['targetingText'] = df_report['targetingText'].map(lambda x: str(x))
             conn.insert_df("AmazonReport.SD_targeting_sdTargetingV3", df_report[params['columns']])
         except Exception as e:
             print('插入出错',e)
             try:
                 conn = self.clickhouse_connect()
+                df_report[df_report[params['columns']].select_dtypes(include=['object']).columns] =  df_report[df_report[params['columns']].select_dtypes(include=['object']).columns].apply(lambda x: x.astype(str))
+                df_report[df_report[params['columns']].select_dtypes(include=['int64']).columns] = df_report[df_report[params['columns']].select_dtypes(include=['int64']).columns].astype('int32')
+                df_report[df_report[params['columns']].select_dtypes(include=['float64']).columns] = df_report[df_report[params['columns']].select_dtypes(include=['float64']).columns].astype('float32')
                 conn.insert_df("AmazonReport.SD_targeting_sdTargetingV3", df_report[params['columns']])
+                print('完成')
             except Exception as e:
                 print('插入出错，重试失败。错误原因: ',e)
-                return ''
-            return ''
+
         time.sleep(1.05)
         print("插入完成SD_targeting_sdTargetingV3")
 
-
         df_rel =  df_report[params['columns']]
         return df_rel
 
@@ -2097,23 +2107,21 @@ class SD_ETL(SDClient, Common_ETLMethod):
                                          endDate=params['endDate'], reportType=params['reportType'])
         # print(list_report)
         df_report = pd.json_normalize(list_report)
-        # print("=====",df_report,sep='\n')
-        # print(df_report.to_excel('temp1.xlsx',index=False))
-        # print(df_report.dtypes)
         df_report = self.type_trans(df_report, params['columns'], timeZone_, extra_columns=[])
 
-        # print(df_report)
         try:
+            df_report['targetingText'] = df_report['targetingText'].fillna('-')
+            df_report['targetingText'] = df_report['targetingText'].map(lambda x: str(x))
             conn.insert_df("AmazonReport.SD_targeting_sdTargetingMatchedTargetV3", df_report[params['columns']])
         except Exception as e:
             print('插入出错',e)
             try:
                 conn = self.clickhouse_connect()
+
                 conn.insert_df("AmazonReport.SD_targeting_sdTargetingMatchedTargetV3", df_report[params['columns']])
             except Exception as e:
                 print('插入出错，重试失败。错误原因: ',e)
-                return ''
-            return ''
+
         time.sleep(1.05)
         print("插入完成SD_targeting_sdTargetingMatchedTargetV3")
 
@@ -2169,7 +2177,6 @@ class SD_ETL(SDClient, Common_ETLMethod):
             except Exception as e:
                 print('插入出错，重试失败。错误原因: ',e)
                 return ''
-            return ''
         time.sleep(1.05)
         print("插入完成SD_advertiser_sdAdvertisedProductV3")
 
@@ -2217,7 +2224,6 @@ class SD_ETL(SDClient, Common_ETLMethod):
             except Exception as e:
                 print('插入出错，重试失败。错误原因: ',e)
                 return ''
-            return ''
         time.sleep(1.05)
         print("插入完成SD_asin_sdPurchasedProductV3")
 
@@ -2276,7 +2282,6 @@ class SD_ETL(SDClient, Common_ETLMethod):
             except Exception as e:
                 print('插入出错，重试失败。错误原因: ',e)
                 return ''
-            return ''
         time.sleep(1.05)
         print("插入完成SD_campaignsV2—")
 
@@ -2318,7 +2323,6 @@ class SD_ETL(SDClient, Common_ETLMethod):
             except Exception as e:
                 print('插入出错，重试失败。错误原因: ',e)
                 return ''
-            return ''
         time.sleep(1.05)
         print("插入完成SD_campaignsV2")
 
@@ -2373,7 +2377,6 @@ class SD_ETL(SDClient, Common_ETLMethod):
             except Exception as e:
                 print('插入出错，重试失败。错误原因: ',e)
                 return ''
-            return ''
         time.sleep(1.05)
         print("插入完成SD_adGroupsV2")
 
@@ -2414,7 +2417,6 @@ class SD_ETL(SDClient, Common_ETLMethod):
             except Exception as e:
                 print('插入出错，重试失败。错误原因: ',e)
                 return ''
-            return ''
         time.sleep(1.05)
         print("插入完成SD_adGroupsV2")
 
@@ -2469,7 +2471,6 @@ class SD_ETL(SDClient, Common_ETLMethod):
             except Exception as e:
                 print('插入出错，重试失败。错误原因: ',e)
                 return ''
-            return ''
         time.sleep(1.05)
         print("插入完成SD_adsV2")
 
@@ -2510,7 +2511,6 @@ class SD_ETL(SDClient, Common_ETLMethod):
             except Exception as e:
                 print('插入出错，重试失败。错误原因: ',e)
                 return ''
-            return ''
         time.sleep(1.05)
         print("插入完成SD_adsV2")
 
@@ -2565,7 +2565,6 @@ class SD_ETL(SDClient, Common_ETLMethod):
             except Exception as e:
                 print('插入出错，重试失败。错误原因: ',e)
                 return ''
-            return ''
         time.sleep(1.05)
         print("插入完成SD_targetsV2")
 
@@ -2606,7 +2605,6 @@ class SD_ETL(SDClient, Common_ETLMethod):
             except Exception as e:
                 print('插入出错，重试失败。错误原因: ',e)
                 return ''
-            return ''
         time.sleep(1.05)
         print("插入完成SD_targetsV2")
 
@@ -2658,7 +2656,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
             except Exception as e:
                 print('插入出错，重试失败。错误原因: ',e)
                 return ''
-            return ''
+
         time.sleep(1.05)
         print("插入完成SD_asinsV2")
 
@@ -2690,7 +2688,6 @@ class SD_ETL(SDClient, Common_ETLMethod):
             except Exception as e:
                 print('插入出错，重试失败。错误原因: ',e)
                 return ''
-            return ''
         time.sleep(1.05)
         print("插入完成SD_asinsV2")
 
@@ -2749,7 +2746,6 @@ class SD_ETL(SDClient, Common_ETLMethod):
             except Exception as e:
                 print('插入出错，重试失败。错误原因: ',e)
                 return ''
-            return ''
         time.sleep(1.05)
         print("插入完成SD_campaignsMatchedTargetV2")
 
@@ -2790,7 +2786,6 @@ class SD_ETL(SDClient, Common_ETLMethod):
             except Exception as e:
                 print('插入出错，重试失败。错误原因: ',e)
                 return ''
-            return ''
         time.sleep(1.05)
         print("插入完成SD_campaignsMatchedTargetV2")
 
@@ -2845,7 +2840,6 @@ class SD_ETL(SDClient, Common_ETLMethod):
             except Exception as e:
                 print('插入出错，重试失败。错误原因: ',e)
                 return ''
-            return ''
         time.sleep(1.05)
         print("插入完成SD_adGroupsMatchedTargetV2")
 
@@ -2884,7 +2878,6 @@ class SD_ETL(SDClient, Common_ETLMethod):
             except Exception as e:
                 print('插入出错，重试失败。错误原因: ',e)
                 return ''
-            return ''
         time.sleep(1.05)
         print("插入完成SD_adGroupsMatchedTargetV2")
 
@@ -2937,7 +2930,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
             except Exception as e:
                 print('插入出错，重试失败。错误原因: ',e)
                 return ''
-            return ''
+
         time.sleep(1.05)
         print("插入完成SD_targetsMatchedTargetV2")
 
@@ -2976,7 +2969,6 @@ class SD_ETL(SDClient, Common_ETLMethod):
             except Exception as e:
                 print('插入出错，重试失败。错误原因: ',e)
                 return ''
-            return ''
         time.sleep(1.05)
         print("插入完成SD_targetsMatchedTargetV2")