Merge branch 'wengao' of ASJ_ADS/sync_amz_data into master

sync_amz_data/DataTransform/Data_ETL.py

@@ -75,8 +75,7 @@ class Common_ETLMethod(BaseClient):
         df.drop(columns=[segment], inplace=True)
         return df
 
-    @classmethod
-    def get_keyOvalue(cls,expressions, result: Literal['value', 'type']):
+    def get_keyOvalue(self,expressions, result: Literal['value', 'type']):
         if expressions not in [None,pd.NA,pd.NaT]:
             if len(expressions) > 1:
                 return [i.get(result) for i in expressions]
@@ -210,13 +209,13 @@ class SP_ETL(SPClient, Common_ETLMethod):
         df_targets = pd.json_normalize(list_targets)
         df_targets = self.TZ_Deal(df_targets, ["extendedData.creationDateTime", "extendedData.lastUpdateDateTime"])
         df_targets['resolvedExpressions_type'] = df_targets['resolvedExpression'].map(
-            lambda x: Common_ETLMethod.get_keyOvalue(x, 'type'))
+            lambda x: self.get_keyOvalue(x, 'type'))
         df_targets['resolvedExpressions_value'] = df_targets['resolvedExpression'].map(
-            lambda x: Common_ETLMethod.get_keyOvalue(x, 'value'))
+            lambda x: self.get_keyOvalue(x, 'value'))
         df_targets['expression_type'] = df_targets['expression'].map(
-            lambda x: Common_ETLMethod.get_keyOvalue(x, 'type'))
+            lambda x: self.get_keyOvalue(x, 'type'))
         df_targets['expression_value'] = df_targets['expression'].map(
-            lambda x: Common_ETLMethod.get_keyOvalue(x, 'value'))
+            lambda x: self.get_keyOvalue(x, 'value'))
         return self.columnsName_modify(df_targets)
 
     def negative_targets_ETL(self):
@@ -224,13 +223,13 @@ class SP_ETL(SPClient, Common_ETLMethod):
         df_targets = pd.json_normalize(list_targets)
         df_targets = self.TZ_Deal(df_targets, ["extendedData.creationDateTime", "extendedData.lastUpdateDateTime"])
         df_targets['resolvedExpressions_type'] = df_targets['resolvedExpression'].map(
-            lambda x: Common_ETLMethod.get_keyOvalue(x, 'type'))
+            lambda x: self.get_keyOvalue(x, 'type'))
         df_targets['resolvedExpressions_value'] = df_targets['resolvedExpression'].map(
-            lambda x: Common_ETLMethod.get_keyOvalue(x, 'value'))
+            lambda x: self.get_keyOvalue(x, 'value'))
         df_targets['expression_type'] = df_targets['expression'].map(
-            lambda x: Common_ETLMethod.get_keyOvalue(x, 'type'))
+            lambda x: self.get_keyOvalue(x, 'type'))
         df_targets['expression_value'] = df_targets['expression'].map(
-            lambda x: Common_ETLMethod.get_keyOvalue(x, 'value'))
+            lambda x: self.get_keyOvalue(x, 'value'))
         return self.columnsName_modify(df_targets)
 
     def budget_ETL(self, campaign_ids: list):
@@ -532,8 +531,12 @@ class SB_ETL(SBClient, Common_ETLMethod):
         # df_targets = self.TZ_Deal(df_targets, ["extendedData.creationDateTime", "extendedData.lastUpdateDateTime"])
         # df_targets = self.expression_split(df_targets, "resolvedExpressions")
         df_targets = self.id_type_trans(df_targets)
-        df_targets['resolvedExpressions_type'] = df_targets['resolvedExpressions'].map(lambda x:Common_ETLMethod.get_keyOvalue(x,'type'))
-        df_targets['resolvedExpressions_value'] = df_targets['resolvedExpressions'].map(lambda x:Common_ETLMethod.get_keyOvalue(x,'value'))
+        df_targets['expressions_type'] = df_targets['expressions'].map(
+            lambda x: self.get_keyOvalue(x, 'type'))
+        df_targets['expressions_value'] = df_targets['expressions'].map(
+            lambda x: self.get_keyOvalue(x, 'value'))
+        df_targets['resolvedExpressions_type'] = df_targets['resolvedExpressions'].map(lambda x:self.get_keyOvalue(x, 'type'))
+        df_targets['resolvedExpressions_value'] = df_targets['resolvedExpressions'].map(lambda x:self.get_keyOvalue(x, 'value'))
         return self.columnsName_modify(df_targets)
 
     def budget_ETL(self, campaign_ids: list):
@@ -547,16 +550,17 @@ class SB_ETL(SBClient, Common_ETLMethod):
         timeZone_,today = self.today_()
         params = self.config_params(params)
         params['reportType'] = "sbCampaigns" #sbCampaigns
-        params['columns'] = ['campaignName','campaignId','campaignStatus',
-                             'campaignBudgetAmount', 'campaignBudgetCurrencyCode', 'campaignBudgetType','impressions',
-                             'clicks', 'cost','addToCart', 'addToCartClicks', 'addToCartRate','brandedSearches','brandedSearchesClicks'
-                             ,'costType','detailPageViews','detailPageViewsClicks','eCPAddToCart','newToBrandDetailPageViewRate',
-                             'newToBrandDetailPageViews', 'newToBrandDetailPageViewsClicks',
-                            'newToBrandECPDetailPageView','newToBrandSales', 'newToBrandSalesClicks','newToBrandPurchases',
-                             'newToBrandPurchasesClicks', 'newToBrandPurchasesPercentage','newToBrandPurchasesRate','newToBrandUnitsSold',
-                             'newToBrandUnitsSoldClicks', 'newToBrandUnitsSoldPercentage','newToBrandSalesPercentage','date', 'purchases',
-                             'purchasesClicks', 'purchasesPromoted', 'sales', 'salesClicks', 'salesPromoted','topOfSearchImpressionShare',
-                             'unitsSold', 'unitsSoldClicks', 'video5SecondViewRate',
+        params['columns'] = ['campaignId',
+                             'campaignName','campaignBudgetAmount', 'campaignBudgetCurrencyCode', 'campaignBudgetType', 'topOfSearchImpressionShare',
+                             'addToCart', 'addToCartClicks', 'addToCartRate', 'brandedSearches', 'brandedSearchesClicks',
+                             'campaignBudgetAmount', 'campaignBudgetCurrencyCode', 'campaignBudgetType',  'campaignStatus', 'clicks', 'cost',
+                             'costType', 'date', 'detailPageViews','detailPageViewsClicks', 'eCPAddToCart', 'endDate', 'impressions', 'newToBrandDetailPageViewRate',
+                             'newToBrandDetailPageViews', 'newToBrandDetailPageViewsClicks', 'newToBrandECPDetailPageView',
+                             'newToBrandPurchases', 'newToBrandPurchasesClicks', 'newToBrandPurchasesPercentage',
+                             'newToBrandPurchasesRate', 'newToBrandSales', 'newToBrandSalesClicks', 'newToBrandSalesPercentage',
+                             'newToBrandUnitsSold', 'newToBrandUnitsSoldClicks', 'newToBrandUnitsSoldPercentage', 'purchases',
+                             'purchasesClicks', 'purchasesPromoted', 'sales', 'salesClicks', 'salesPromoted', 'startDate',
+                             'topOfSearchImpressionShare', 'unitsSold', 'unitsSoldClicks', 'video5SecondViewRate',
                              'video5SecondViews', 'videoCompleteViews', 'videoFirstQuartileViews', 'videoMidpointViews',
                              'videoThirdQuartileViews', 'videoUnmutes', 'viewabilityRate', 'viewableImpressions',
                              'viewClickThroughRate'
@@ -571,169 +575,14 @@ class SB_ETL(SBClient, Common_ETLMethod):
         df_report = self.type_trans(df_report, params['columns'], timeZone_, extra_columns=[])
 
         # print(df_report)
-        conn.insert_df("AmazonReport.SB_sbCampaigns_campaignV3", df_report[params['columns']])
-        time.sleep(0.05)
-        print("插入完成SB_sbCampaigns_campaignV3")
-        df_rel =  df_report[params['columns']]
-        return df_rel
-
-    def reportV3_adGroup_sbAdGroup_ETL(self, conn, params:dict=None):
-        print(params)
-        timeZone_,today = self.today_()
-        params = self.config_params(params)
-        params['reportType'] = "sbAdGroup" #sbCampaigns
-        params['columns'] = ['campaignName','campaignId','campaignBudgetCurrencyCode','adGroupName','adGroupId', 'impressions',  'clicks', 'cost',
-                                'addToCartRate',  'brandedSearches', 'brandedSearchesClicks','detailPageViews', 'detailPageViewsClicks',
-                                    'eCPAddToCart','newToBrandSales', 'newToBrandSalesClicks','newToBrandPurchases', 'newToBrandPurchasesClicks',
-                                  'newToBrandUnitsSold', 'newToBrandUnitsSoldClicks','date',
-                                 'purchases', 'purchasesClicks',  'sales', 'salesClicks',
-                                'unitsSold', 'unitsSoldClicks', 'videoCompleteViews',
-                                 'videoFirstQuartileViews', 'videoMidpointViews', 'videoThirdQuartileViews', 'videoUnmutes',
-                                 'viewabilityRate', 'viewClickThroughRate'
-        ]  # 'startDate', 'endDate',
-        params['groupby'] = ['adGroup']
-        params['timeUnit'] = 'DAILY'
-        list_report = self.get_v3_report(timeUnit=params['timeUnit'], groupby=params['groupby'],
-                                         columns=params['columns'], startDate=params['startDate'],
-                                         endDate=params['endDate'], reportType=params['reportType'])
-        # print(list_report)
-        df_report = pd.json_normalize(list_report)
-        df_report = self.type_trans(df_report, params['columns'], timeZone_, extra_columns=[])
-
-        # print(df_report)
-        conn.insert_df("AmazonReport.SB_sbAdGroup_adGroupV3", df_report[params['columns']])
-        time.sleep(0.05)
-        print("插入完成SB_sbAdGroup_adGroupV3")
-        df_rel =  df_report[params['columns']]
-        return df_rel
-
-    def reportV3_sbCampaignPlacement_ETL(self, conn, params:dict=None):
-        print(params)
-        timeZone_,today = self.today_()
-        params = self.config_params(params)
-        params['reportType'] = "sbCampaignPlacement" #sbCampaigns
-        params['columns'] = ['placementClassification','campaignName','campaignId','campaignStatus','campaignBudgetAmount','campaignBudgetType','campaignBudgetCurrencyCode',
-                            'impressions','clicks', 'cost','addToCart', 'addToCartClicks', 'addToCartRate', 'brandedSearches',
-                            'brandedSearchesClicks', 'costType','detailPageViews','detailPageViewsClicks', 'eCPAddToCart', 'newToBrandDetailPageViewRate',
-                            'newToBrandDetailPageViews', 'newToBrandDetailPageViewsClicks','newToBrandECPDetailPageView','newToBrandSales', 'newToBrandSalesClicks',
-                            'newToBrandPurchases', 'newToBrandPurchasesClicks','newToBrandPurchasesPercentage', 'newToBrandPurchasesRate',
-                            'newToBrandUnitsSold','newToBrandUnitsSoldClicks', 'newToBrandUnitsSoldPercentage','newToBrandSalesPercentage',
-                            'date',  'purchases', 'purchasesClicks', 'purchasesPromoted',  'sales', 'salesClicks', 'salesPromoted',
-                            'unitsSold', 'unitsSoldClicks', 'video5SecondViewRate', 'video5SecondViews','videoCompleteViews', 'videoFirstQuartileViews',
-                            'videoMidpointViews', 'videoThirdQuartileViews', 'videoUnmutes','viewabilityRate', 'viewableImpressions', 'viewClickThroughRate'
-        ]  # 'startDate', 'endDate',
-        params['groupby'] = ['campaignPlacement']
-        params['timeUnit'] = 'DAILY'
-        list_report = self.get_v3_report(timeUnit=params['timeUnit'], groupby=params['groupby'],
-                                         columns=params['columns'], startDate=params['startDate'],
-                                         endDate=params['endDate'], reportType=params['reportType'])
-        # print(list_report)
-        df_report = pd.json_normalize(list_report)
-        df_report = self.type_trans(df_report, params['columns'], timeZone_, extra_columns=[])
-
-        # print(df_report)
-        conn.insert_df("AmazonReport.SB_sbCampaigns_placementV3", df_report[params['columns']])
-        time.sleep(0.05)
-        print("插入完成SB_sbCampaigns_placementV3")
-        df_rel =  df_report[params['columns']]
-        return df_rel
-
-    def reportV3_sbTargeting_ETL(self, conn, params:dict=None):
-        print(params)
-        timeZone_,today = self.today_()
-        params = self.config_params(params)
-        params['reportType'] = "sbTargeting" #sbCampaigns
-        params['columns'] = [
-            'campaignName','campaignId','campaignStatus','campaignBudgetAmount','campaignBudgetType','campaignBudgetCurrencyCode',
-            'adGroupName','adGroupId','keywordText','keywordId', 'adKeywordStatus',  'keywordType', 'matchType','keywordBid',
-            'targetingText', 'targetingId', 'targetingType', 'impressions', 'clicks', 'cost',
-            'addToCart', 'addToCartClicks', 'addToCartRate',   'brandedSearches', 'brandedSearchesClicks',
-            'costType', 'detailPageViews', 'detailPageViewsClicks','eCPAddToCart','newToBrandDetailPageViewRate',
-            'newToBrandDetailPageViews','newToBrandDetailPageViewsClicks','newToBrandECPDetailPageView','newToBrandSales',
-            'newToBrandSalesClicks','newToBrandPurchases', 'newToBrandPurchasesClicks',
-            'newToBrandPurchasesPercentage', 'newToBrandPurchasesRate',  'newToBrandUnitsSold', 'newToBrandUnitsSoldClicks', 'newToBrandUnitsSoldPercentage',
-            'newToBrandSalesPercentage','date',    'purchases', 'purchasesClicks', 'purchasesPromoted',
-            'sales', 'salesClicks', 'salesPromoted', 'targetingExpression',  'topOfSearchImpressionShare'
-        ]  # 'startDate', 'endDate',
-        params['groupby'] = ['targeting']
-        params['timeUnit'] = 'DAILY'
-        list_report = self.get_v3_report(timeUnit=params['timeUnit'], groupby=params['groupby'],
-                                         columns=params['columns'], startDate=params['startDate'],
-                                         endDate=params['endDate'], reportType=params['reportType'])
-        # print(list_report)
-        df_report = pd.json_normalize(list_report)
-        df_report = self.type_trans(df_report, params['columns'], timeZone_, extra_columns=[])
+        # conn.insert_df("AmazonReport.SB_sbPurchasedProduct_asinV3", df_report[params['columns']])
+        # time.sleep(0.05)
+        # print("插入完成SB_sbPurchasedProduct_asinV3")
 
-        # print(df_report)
-        conn.insert_df("AmazonReport.SB_sbTargeting_targetingV3", df_report[params['columns']])
-        time.sleep(0.05)
-        print("插入完成SB_sbTargeting_targetingV3")
-        df_rel =  df_report[params['columns']]
-        return df_rel
 
-    def reportV3_sbSearchTerm_ETL(self, conn, params:dict=None):
-        print(params)
-        timeZone_,today = self.today_()
-        params = self.config_params(params)
-        params['reportType'] = "sbSearchTerm" #sbCampaigns
-        params['columns'] = [
-            'campaignName','campaignId','campaignStatus','campaignBudgetAmount','campaignBudgetType', 'campaignBudgetCurrencyCode', 'adGroupName',
-            'adGroupId',  'keywordText','keywordId','adKeywordStatus', 'matchType','keywordBid','impressions',  'clicks', 'cost', 'costType', 'date',
-             'purchases', 'purchasesClicks', 'sales', 'salesClicks', 'searchTerm', 'unitsSold',
-            'video5SecondViewRate', 'video5SecondViews', 'videoCompleteViews', 'videoFirstQuartileViews', 'videoMidpointViews',
-            'videoThirdQuartileViews', 'videoUnmutes', 'viewabilityRate', 'viewableImpressions', 'viewClickThroughRate'
-        ]  # 'startDate', 'endDate',
-        params['groupby'] = ['searchTerm']
-        params['timeUnit'] = 'DAILY'
-        list_report = self.get_v3_report(timeUnit=params['timeUnit'], groupby=params['groupby'],
-                                         columns=params['columns'], startDate=params['startDate'],
-                                         endDate=params['endDate'], reportType=params['reportType'])
-        # print(list_report)
-        df_report = pd.json_normalize(list_report)
-        df_report = self.type_trans(df_report, params['columns'], timeZone_, extra_columns=[])
-
-        # print(df_report)
-        conn.insert_df("AmazonReport.SB_sbSearchTerm_searchTermV3", df_report[params['columns']])
-        time.sleep(0.05)
-        print("插入完成SB_sbSearchTerm_searchTermV3")
         df_rel =  df_report[params['columns']]
         return df_rel
 
-    def reportV3_sbAds_ETL(self, conn, params:dict=None):
-        print(params)
-        timeZone_,today = self.today_()
-        params = self.config_params(params)
-        params['reportType'] = "sbAds" #sbCampaigns
-        params['columns'] = [
-            'campaignName', 'campaignId','campaignStatus','campaignBudgetAmount','campaignBudgetCurrencyCode', 'campaignBudgetType',
-            'adGroupName','adGroupId', 'adId','impressions','clicks', 'cost',
-            'addToCart', 'addToCartClicks', 'addToCartRate',   'brandedSearches',
-            'brandedSearchesClicks',  'costType','detailPageViews', 'detailPageViewsClicks','eCPAddToCart',
-            'newToBrandDetailPageViewRate', 'newToBrandDetailPageViews','newToBrandDetailPageViewsClicks','newToBrandECPDetailPageView',
-            'newToBrandSales','newToBrandSalesClicks', 'newToBrandPurchases',
-            'newToBrandPurchasesClicks', 'newToBrandPurchasesPercentage', 'newToBrandPurchasesRate',  'newToBrandUnitsSold', 'newToBrandUnitsSoldClicks',
-            'newToBrandUnitsSoldPercentage','newToBrandSalesPercentage','date','purchases', 'purchasesClicks', 'purchasesPromoted', 'sales', 'salesClicks',
-            'salesPromoted',  'unitsSold', 'unitsSoldClicks', 'video5SecondViewRate', 'video5SecondViews',
-            'videoCompleteViews', 'videoFirstQuartileViews', 'videoMidpointViews', 'videoThirdQuartileViews', 'videoUnmutes',
-            'viewabilityRate', 'viewableImpressions'
-        ]  # 'startDate', 'endDate',
-        params['groupby'] = ['ads']
-        params['timeUnit'] = 'DAILY'
-        list_report = self.get_v3_report(timeUnit=params['timeUnit'], groupby=params['groupby'],
-                                         columns=params['columns'], startDate=params['startDate'],
-                                         endDate=params['endDate'], reportType=params['reportType'])
-        # print(list_report)
-        df_report = pd.json_normalize(list_report)
-        df_report['date'] = df_report['date'].map(lambda x: datetime.fromtimestamp(int(x)/1000).date(),na_action='ignore')#,na_action='ignore'
-        df_report = self.type_trans(df_report, params['columns'], timeZone_, extra_columns=[])
-
-        # print(df_report)
-        conn.insert_df("AmazonReport.SB_sbAds_adsV3", df_report[params['columns']])
-        time.sleep(0.05)
-        print("插入完成SB_sbAds_adsV3")
-
-        df_rel =  df_report[params['columns']]
-        return df_rel
 
     def reportV3_purchasedAsinRecord_ETL(self, conn, params:dict=None):
         print(params)
@@ -1240,8 +1089,8 @@ class SD_ETL(SDClient, Common_ETLMethod):
         df_targets = pd.json_normalize(list_targets)
         # df_targets = self.expression_split(df_targets, "resolvedExpression")
         df_targets = self.id_type_trans(df_targets)
-        df_targets['resolvedExpressions_type'] = df_targets['resolvedExpression'].map(lambda x:Common_ETLMethod.get_keyOvalue(x,'type'))
-        df_targets['resolvedExpressions_value'] = df_targets['resolvedExpression'].map(lambda x:Common_ETLMethod.get_keyOvalue(x,'value'))
+        df_targets['resolvedExpressions_type'] = df_targets['resolvedExpression'].map(lambda x:self.get_keyOvalue(x,'type'))
+        df_targets['resolvedExpressions_value'] = df_targets['resolvedExpression'].map(lambda x:self.get_keyOvalue(x,'value'))
         return self.columnsName_modify(df_targets)
 
     def budget_ETL(self, campaignsIds: list):
@@ -2027,15 +1876,59 @@ if __name__ == '__main__':
     }
     conn = Common_ETLMethod(**AWS_CREDENTIALS).clickhouse_connect()
 
-    sb_ = SP_ETL(**AWS_CREDENTIALS)
-    print(sb_.targets_ETL())
-    # list_date = ['2023-11-20']
-    # list_date = [f'2023-11-{"0" + str(i) if len(str(i)) == 1 else i}' for i in range(1, 22)]
-    # for date_ in list_date:
-    #     print(date_)
-    #     print(date_.replace("-",""))
-    #     sb_ = SB_ETL(**AWS_CREDENTIALS)
-    #     rel = sb_.reportV3_sbAds_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-    #     print(rel)
-    #     print(rel.info())
-    #     print(rel.columns)
+    # SD
+    ac_etl = SP_ETL(**AWS_CREDENTIALS)
+    # ls = ac_etl.reportV2_campaignsRecord_t2_ETL(conn)
+    ls = ac_etl.targets_ETL()
+    print(ls)
+    # ls.to_csv('temp.csv')
+    # print(ls)
+    # print(ls.info())
+    # print(ls.to_excel('obse11.xlsx'))
+    # ac_etl.reportV2_campaignsRecord_t2_ETL(conn)
+    # ac_etl.reportV2_campaignsRecord_t3_ETL(conn)
+    # ac_etl.reportV2_adGroupsRecord_t2_ETL(conn)
+    # ac_etl.reportV2_adGroupsRecord_t3_ETL(conn)
+    # ac_etl.reportV2_asins_t2_ETL(conn)
+    # ac_etl.reportV2_asins_t3_ETL(conn)
+    # ac_etl.reportV2_productAds_t2_ETL(conn)
+    # ac_etl.reportV2_productAds_t3_ETL(conn)
+    # ac_etl.reportV2_targets_t2_ETL(conn)
+    # ac_etl.reportV2_productAds_t3_ETL(conn)
+    # ac_etl.reportV2_campaign_matchedTarget_t2_ETL(conn)
+    # ac_etl.reportV2_campaign_matchedTarget_t3_ETL(conn)
+    # ac_etl.reportV2_adGroups_matchedTarget_t2_ETL(conn)
+    # ac_etl.reportV2_adGroups_matchedTarget_t3_ETL(conn)
+    # ac_etl.reportV2_targets_matchedTarget_t2_ETL(conn)
+    # ac_etl.reportV2_targets_matchedTarget_t3_ETL(conn)
+
+    # SB
+    # ac_etl = SB_ETL(**AWS_CREDENTIALS)
+    # ac_etl.reportV3_purchasedAsinRecord_ETL(conn)
+    # ac_etl.reportV2_campaignsRecord_ETL(conn)
+    # ac_etl.reportV2_campaignsVideo_ETL(conn)
+    # ac_etl.reportV2_adGroupsRecord_ETL(conn)
+    # ac_etl.reportV2_adGroupsVideo_ETL(conn)
+    # ac_etl.reportV2_adsRecord_ETL(conn)
+    # ac_etl.reportV2_adsVideo_ETL(conn)
+    # ac_etl.reportV2_keywordsRecord_ETL(conn)
+    # ac_etl.reportV2_keywordsVideo_ETL(conn)
+    # ac_etl.reportV2_placementRecord_ETL(conn)
+    # ac_etl.reportV2_placementVideo_ETL(conn)
+    # ac_etl.reportV2_searchtermsRecord_ETL(conn)
+    # ac_etl.reportV2_searchtermsVideo_ETL(conn)
+    # ac_etl.reportV2_targetsRecord_ETL(conn)
+    # ac_etl.reportV2_targetsVideo_ETL(conn)
+
+    # SP
+    # ac_etl = SP_ETL(**AWS_CREDENTIALS)
+    # ac_etl.reportV3_campaign_spCampaignsETL(conn)
+    # ac_etl.reportV3_adGroup_spCampaignsETL(conn)
+    # ac_etl.reportV3_campaignPlacement_spCampaignsETL(conn)
+    # ac_etl.reportV3_targeting_spTargetingETL(conn)
+    # ac_etl.reportV3_searchTerm_spSearchTermETL(conn)
+    # ac_etl.reportV3_advertiser_spAdvertisedProductETL(conn)
+    # ac_etl.reportV3_asin_spPurchasedProductETL(conn)
+
+    conn.close()
+    ###

sync_amz_data/tasks/datainsert/SB/mysql_datainsert_sbtarget.py

@@ -0,0 +1,88 @@
+from sync_amz_data.DataTransform.Data_ETL import SB_ETL
+import requests
+from urllib.parse import urljoin
+from sync_amz_data.settings import AWS_LWA_CLIENT
+import pandas as pd
+import json
+
+
+class RateLimitError(Exception):
+    def __init__(self, retry_after: str = None):
+        self.retry_after = retry_after
+
+
+def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None):
+    ADS = "http://192.168.1.23:8001/"
+    resp = requests.session().request(
+        method=method,
+        url=urljoin(ADS, url_path),
+        headers=head,
+        params=params,
+        json=body,
+    )
+    if resp.status_code == 429:
+        raise RateLimitError(resp.headers.get("Retry-After"))
+    if resp.status_code >= 400:
+        raise Exception(resp.text)
+    return resp.json()
+
+
+class SbTargets:
+    def __init__(self, profile_id):
+        self.profile_id = profile_id
+        self.re_url_path = "api/ad_manage/profiles/"
+        self.upcreate_url_path = "api/ad_manage/sbtargets/updata/"
+        self.heads = {'X-Token': "da4ab6bc5cbf1dfa"}
+        self.refresh_token = self.get_refresh_token()
+        self.lwa_client_id = AWS_LWA_CLIENT['lwa_client_id']
+        self.lwa_client_secret = AWS_LWA_CLIENT['lwa_client_secret']
+        self.AWS_CREDENTIALS = {
+            'lwa_client_id': self.lwa_client_id,
+            'lwa_client_secret': self.lwa_client_secret,
+            'refresh_token': self.refresh_token,
+            'profile_id': self.profile_id
+        }
+
+    def get_refresh_token(self):
+        params = {'profile_id': self.profile_id}
+        heads = self.heads
+        url_path = self.re_url_path
+        tem = request(url_path=url_path, head=heads, params=params)
+        if tem.get('data') is not None:
+            _ = tem.get('data')
+            out = _[0].get('refresh_token')
+        else:
+            out = None
+        return out
+
+    def get_sbtargets_data(self):
+        tem = SB_ETL(**self.AWS_CREDENTIALS)
+        df = tem.targets_ETL()
+        if len(df) > 0:
+            df.rename(columns={'adgroupid': 'adGroup',
+                               'campaignid': 'campaign',
+                               'targetid': 'targetId',
+                               'expressions_type': 'expression_type',
+                               'expressions_value': 'expression_value',
+                               'resolvedexpressions_type': 'resolvedExpression_type',
+                               'resolvedexpressions_value': 'resolvedExpression_value'}, inplace=True)
+            df.drop(columns=['expressions', 'resolvedexpressions'], inplace=True)
+            df['profile'] = self.profile_id
+        return df
+
+    def updata_create(self):
+        df_data = self.get_sbtargets_data()
+        _ = df_data.to_json(orient='records', date_format='iso')
+        body = json.loads(_)
+        heads = self.heads
+        url_path = self.upcreate_url_path
+        tem = request(url_path=url_path, head=heads, body=body, method="POST")
+        return tem
+
+
+
+if __name__ == '__main__':
+    a = SbTargets(profile_id="3006125408623189")
+    # out = a.get_sbtargets_data()
+    out = a.updata_create()
+    print(out)

sync_amz_data/tasks/datainsert/alldata_insert.py

@@ -15,7 +15,7 @@ from SB.mysql_datainsert_sbgroup import SbGroup
 from SB.mysql_datainsert_sbkeyword_v3 import SbKeyword
 from SB.mysql_datainsert_sbads import SbAds
 from SB.mysql_datainsert_sb_keywordsbid_recommendations import SbkeywordsBidRecommendations
-
+from SB.mysql_datainsert_sbtarget import SbTargets
 
 pf = Portfolios("3006125408623189")
 pfo = pf.updata_create()
@@ -81,3 +81,9 @@ print("SbAds", sbao)
 sbkbr = SbkeywordsBidRecommendations(profile_id="3006125408623189")
 sbkbro = sbkbr.updata_create()
 print("SbkeywordsBidRecommendations", sbkbro)
+
+sbt = SbTargets(profile_id="3006125408623189")
+sbto = sbt.updata_create()
+print("SbTargets", sbto)
+
+