|
@@ -546,17 +546,16 @@ class SB_ETL(SBClient, Common_ETLMethod):
|
|
|
timeZone_,today = self.today_()
|
|
|
params = self.config_params(params)
|
|
|
params['reportType'] = "sbCampaigns" #sbCampaigns
|
|
|
- params['columns'] = ['campaignId',
|
|
|
- 'campaignName','campaignBudgetAmount', 'campaignBudgetCurrencyCode', 'campaignBudgetType', 'topOfSearchImpressionShare',
|
|
|
- 'addToCart', 'addToCartClicks', 'addToCartRate', 'brandedSearches', 'brandedSearchesClicks',
|
|
|
- 'campaignBudgetAmount', 'campaignBudgetCurrencyCode', 'campaignBudgetType', 'campaignStatus', 'clicks', 'cost',
|
|
|
- 'costType', 'date', 'detailPageViews','detailPageViewsClicks', 'eCPAddToCart', 'endDate', 'impressions', 'newToBrandDetailPageViewRate',
|
|
|
- 'newToBrandDetailPageViews', 'newToBrandDetailPageViewsClicks', 'newToBrandECPDetailPageView',
|
|
|
- 'newToBrandPurchases', 'newToBrandPurchasesClicks', 'newToBrandPurchasesPercentage',
|
|
|
- 'newToBrandPurchasesRate', 'newToBrandSales', 'newToBrandSalesClicks', 'newToBrandSalesPercentage',
|
|
|
- 'newToBrandUnitsSold', 'newToBrandUnitsSoldClicks', 'newToBrandUnitsSoldPercentage', 'purchases',
|
|
|
- 'purchasesClicks', 'purchasesPromoted', 'sales', 'salesClicks', 'salesPromoted', 'startDate',
|
|
|
- 'topOfSearchImpressionShare', 'unitsSold', 'unitsSoldClicks', 'video5SecondViewRate',
|
|
|
+ params['columns'] = ['campaignName','campaignId','campaignStatus',
|
|
|
+ 'campaignBudgetAmount', 'campaignBudgetCurrencyCode', 'campaignBudgetType','impressions',
|
|
|
+ 'clicks', 'cost','addToCart', 'addToCartClicks', 'addToCartRate','brandedSearches','brandedSearchesClicks'
|
|
|
+ ,'costType','detailPageViews','detailPageViewsClicks','eCPAddToCart','newToBrandDetailPageViewRate',
|
|
|
+ 'newToBrandDetailPageViews', 'newToBrandDetailPageViewsClicks',
|
|
|
+ 'newToBrandECPDetailPageView','newToBrandSales', 'newToBrandSalesClicks','newToBrandPurchases',
|
|
|
+ 'newToBrandPurchasesClicks', 'newToBrandPurchasesPercentage','newToBrandPurchasesRate','newToBrandUnitsSold',
|
|
|
+ 'newToBrandUnitsSoldClicks', 'newToBrandUnitsSoldPercentage','newToBrandSalesPercentage','date', 'purchases',
|
|
|
+ 'purchasesClicks', 'purchasesPromoted', 'sales', 'salesClicks', 'salesPromoted','topOfSearchImpressionShare',
|
|
|
+ 'unitsSold', 'unitsSoldClicks', 'video5SecondViewRate',
|
|
|
'video5SecondViews', 'videoCompleteViews', 'videoFirstQuartileViews', 'videoMidpointViews',
|
|
|
'videoThirdQuartileViews', 'videoUnmutes', 'viewabilityRate', 'viewableImpressions',
|
|
|
'viewClickThroughRate'
|
|
@@ -571,14 +570,169 @@ class SB_ETL(SBClient, Common_ETLMethod):
|
|
|
df_report = self.type_trans(df_report, params['columns'], timeZone_, extra_columns=[])
|
|
|
|
|
|
# print(df_report)
|
|
|
- # conn.insert_df("AmazonReport.SB_sbPurchasedProduct_asinV3", df_report[params['columns']])
|
|
|
- # time.sleep(0.05)
|
|
|
- # print("插入完成SB_sbPurchasedProduct_asinV3")
|
|
|
+ conn.insert_df("AmazonReport.SB_sbCampaigns_campaignV3", df_report[params['columns']])
|
|
|
+ time.sleep(0.05)
|
|
|
+ print("插入完成SB_sbCampaigns_campaignV3")
|
|
|
+ df_rel = df_report[params['columns']]
|
|
|
+ return df_rel
|
|
|
+
|
|
|
+ def reportV3_adGroup_sbAdGroup_ETL(self, conn, params:dict=None):
|
|
|
+ print(params)
|
|
|
+ timeZone_,today = self.today_()
|
|
|
+ params = self.config_params(params)
|
|
|
+ params['reportType'] = "sbAdGroup" #sbCampaigns
|
|
|
+ params['columns'] = ['campaignName','campaignId','campaignBudgetCurrencyCode','adGroupName','adGroupId', 'impressions', 'clicks', 'cost',
|
|
|
+ 'addToCartRate', 'brandedSearches', 'brandedSearchesClicks','detailPageViews', 'detailPageViewsClicks',
|
|
|
+ 'eCPAddToCart','newToBrandSales', 'newToBrandSalesClicks','newToBrandPurchases', 'newToBrandPurchasesClicks',
|
|
|
+ 'newToBrandUnitsSold', 'newToBrandUnitsSoldClicks','date',
|
|
|
+ 'purchases', 'purchasesClicks', 'sales', 'salesClicks',
|
|
|
+ 'unitsSold', 'unitsSoldClicks', 'videoCompleteViews',
|
|
|
+ 'videoFirstQuartileViews', 'videoMidpointViews', 'videoThirdQuartileViews', 'videoUnmutes',
|
|
|
+ 'viewabilityRate', 'viewClickThroughRate'
|
|
|
+ ] # 'startDate', 'endDate',
|
|
|
+ params['groupby'] = ['adGroup']
|
|
|
+ params['timeUnit'] = 'DAILY'
|
|
|
+ list_report = self.get_v3_report(timeUnit=params['timeUnit'], groupby=params['groupby'],
|
|
|
+ columns=params['columns'], startDate=params['startDate'],
|
|
|
+ endDate=params['endDate'], reportType=params['reportType'])
|
|
|
+ # print(list_report)
|
|
|
+ df_report = pd.json_normalize(list_report)
|
|
|
+ df_report = self.type_trans(df_report, params['columns'], timeZone_, extra_columns=[])
|
|
|
+
|
|
|
+ # print(df_report)
|
|
|
+ conn.insert_df("AmazonReport.SB_sbAdGroup_adGroupV3", df_report[params['columns']])
|
|
|
+ time.sleep(0.05)
|
|
|
+ print("插入完成SB_sbAdGroup_adGroupV3")
|
|
|
+ df_rel = df_report[params['columns']]
|
|
|
+ return df_rel
|
|
|
+
|
|
|
+ def reportV3_sbCampaignPlacement_ETL(self, conn, params:dict=None):
|
|
|
+ print(params)
|
|
|
+ timeZone_,today = self.today_()
|
|
|
+ params = self.config_params(params)
|
|
|
+ params['reportType'] = "sbCampaignPlacement" #sbCampaigns
|
|
|
+ params['columns'] = ['placementClassification','campaignName','campaignId','campaignStatus','campaignBudgetAmount','campaignBudgetType','campaignBudgetCurrencyCode',
|
|
|
+ 'impressions','clicks', 'cost','addToCart', 'addToCartClicks', 'addToCartRate', 'brandedSearches',
|
|
|
+ 'brandedSearchesClicks', 'costType','detailPageViews','detailPageViewsClicks', 'eCPAddToCart', 'newToBrandDetailPageViewRate',
|
|
|
+ 'newToBrandDetailPageViews', 'newToBrandDetailPageViewsClicks','newToBrandECPDetailPageView','newToBrandSales', 'newToBrandSalesClicks',
|
|
|
+ 'newToBrandPurchases', 'newToBrandPurchasesClicks','newToBrandPurchasesPercentage', 'newToBrandPurchasesRate',
|
|
|
+ 'newToBrandUnitsSold','newToBrandUnitsSoldClicks', 'newToBrandUnitsSoldPercentage','newToBrandSalesPercentage',
|
|
|
+ 'date', 'purchases', 'purchasesClicks', 'purchasesPromoted', 'sales', 'salesClicks', 'salesPromoted',
|
|
|
+ 'unitsSold', 'unitsSoldClicks', 'video5SecondViewRate', 'video5SecondViews','videoCompleteViews', 'videoFirstQuartileViews',
|
|
|
+ 'videoMidpointViews', 'videoThirdQuartileViews', 'videoUnmutes','viewabilityRate', 'viewableImpressions', 'viewClickThroughRate'
|
|
|
+ ] # 'startDate', 'endDate',
|
|
|
+ params['groupby'] = ['campaignPlacement']
|
|
|
+ params['timeUnit'] = 'DAILY'
|
|
|
+ list_report = self.get_v3_report(timeUnit=params['timeUnit'], groupby=params['groupby'],
|
|
|
+ columns=params['columns'], startDate=params['startDate'],
|
|
|
+ endDate=params['endDate'], reportType=params['reportType'])
|
|
|
+ # print(list_report)
|
|
|
+ df_report = pd.json_normalize(list_report)
|
|
|
+ df_report = self.type_trans(df_report, params['columns'], timeZone_, extra_columns=[])
|
|
|
+
|
|
|
+ # print(df_report)
|
|
|
+ conn.insert_df("AmazonReport.SB_sbCampaigns_placementV3", df_report[params['columns']])
|
|
|
+ time.sleep(0.05)
|
|
|
+ print("插入完成SB_sbCampaigns_placementV3")
|
|
|
+ df_rel = df_report[params['columns']]
|
|
|
+ return df_rel
|
|
|
+
|
|
|
+ def reportV3_sbTargeting_ETL(self, conn, params:dict=None):
|
|
|
+ print(params)
|
|
|
+ timeZone_,today = self.today_()
|
|
|
+ params = self.config_params(params)
|
|
|
+ params['reportType'] = "sbTargeting" #sbCampaigns
|
|
|
+ params['columns'] = [
|
|
|
+ 'campaignName','campaignId','campaignStatus','campaignBudgetAmount','campaignBudgetType','campaignBudgetCurrencyCode',
|
|
|
+ 'adGroupName','adGroupId','keywordText','keywordId', 'adKeywordStatus', 'keywordType', 'matchType','keywordBid',
|
|
|
+ 'targetingText', 'targetingId', 'targetingType', 'impressions', 'clicks', 'cost',
|
|
|
+ 'addToCart', 'addToCartClicks', 'addToCartRate', 'brandedSearches', 'brandedSearchesClicks',
|
|
|
+ 'costType', 'detailPageViews', 'detailPageViewsClicks','eCPAddToCart','newToBrandDetailPageViewRate',
|
|
|
+ 'newToBrandDetailPageViews','newToBrandDetailPageViewsClicks','newToBrandECPDetailPageView','newToBrandSales',
|
|
|
+ 'newToBrandSalesClicks','newToBrandPurchases', 'newToBrandPurchasesClicks',
|
|
|
+ 'newToBrandPurchasesPercentage', 'newToBrandPurchasesRate', 'newToBrandUnitsSold', 'newToBrandUnitsSoldClicks', 'newToBrandUnitsSoldPercentage',
|
|
|
+ 'newToBrandSalesPercentage','date', 'purchases', 'purchasesClicks', 'purchasesPromoted',
|
|
|
+ 'sales', 'salesClicks', 'salesPromoted', 'targetingExpression', 'topOfSearchImpressionShare'
|
|
|
+ ] # 'startDate', 'endDate',
|
|
|
+ params['groupby'] = ['targeting']
|
|
|
+ params['timeUnit'] = 'DAILY'
|
|
|
+ list_report = self.get_v3_report(timeUnit=params['timeUnit'], groupby=params['groupby'],
|
|
|
+ columns=params['columns'], startDate=params['startDate'],
|
|
|
+ endDate=params['endDate'], reportType=params['reportType'])
|
|
|
+ # print(list_report)
|
|
|
+ df_report = pd.json_normalize(list_report)
|
|
|
+ df_report = self.type_trans(df_report, params['columns'], timeZone_, extra_columns=[])
|
|
|
|
|
|
+ # print(df_report)
|
|
|
+ conn.insert_df("AmazonReport.SB_sbTargeting_targetingV3", df_report[params['columns']])
|
|
|
+ time.sleep(0.05)
|
|
|
+ print("插入完成SB_sbTargeting_targetingV3")
|
|
|
+ df_rel = df_report[params['columns']]
|
|
|
+ return df_rel
|
|
|
|
|
|
+ def reportV3_sbSearchTerm_ETL(self, conn, params:dict=None):
|
|
|
+ print(params)
|
|
|
+ timeZone_,today = self.today_()
|
|
|
+ params = self.config_params(params)
|
|
|
+ params['reportType'] = "sbSearchTerm" #sbCampaigns
|
|
|
+ params['columns'] = [
|
|
|
+ 'campaignName','campaignId','campaignStatus','campaignBudgetAmount','campaignBudgetType', 'campaignBudgetCurrencyCode', 'adGroupName',
|
|
|
+ 'adGroupId', 'keywordText','keywordId','adKeywordStatus', 'matchType','keywordBid','impressions', 'clicks', 'cost', 'costType', 'date',
|
|
|
+ 'purchases', 'purchasesClicks', 'sales', 'salesClicks', 'searchTerm', 'unitsSold',
|
|
|
+ 'video5SecondViewRate', 'video5SecondViews', 'videoCompleteViews', 'videoFirstQuartileViews', 'videoMidpointViews',
|
|
|
+ 'videoThirdQuartileViews', 'videoUnmutes', 'viewabilityRate', 'viewableImpressions', 'viewClickThroughRate'
|
|
|
+ ] # 'startDate', 'endDate',
|
|
|
+ params['groupby'] = ['searchTerm']
|
|
|
+ params['timeUnit'] = 'DAILY'
|
|
|
+ list_report = self.get_v3_report(timeUnit=params['timeUnit'], groupby=params['groupby'],
|
|
|
+ columns=params['columns'], startDate=params['startDate'],
|
|
|
+ endDate=params['endDate'], reportType=params['reportType'])
|
|
|
+ # print(list_report)
|
|
|
+ df_report = pd.json_normalize(list_report)
|
|
|
+ df_report = self.type_trans(df_report, params['columns'], timeZone_, extra_columns=[])
|
|
|
+
|
|
|
+ # print(df_report)
|
|
|
+ conn.insert_df("AmazonReport.SB_sbSearchTerm_searchTermV3", df_report[params['columns']])
|
|
|
+ time.sleep(0.05)
|
|
|
+ print("插入完成SB_sbSearchTerm_searchTermV3")
|
|
|
df_rel = df_report[params['columns']]
|
|
|
return df_rel
|
|
|
|
|
|
+ def reportV3_sbAds_ETL(self, conn, params:dict=None):
|
|
|
+ print(params)
|
|
|
+ timeZone_,today = self.today_()
|
|
|
+ params = self.config_params(params)
|
|
|
+ params['reportType'] = "sbAds" #sbCampaigns
|
|
|
+ params['columns'] = [
|
|
|
+ 'campaignName', 'campaignId','campaignStatus','campaignBudgetAmount','campaignBudgetCurrencyCode', 'campaignBudgetType',
|
|
|
+ 'adGroupName','adGroupId', 'adId','impressions','clicks', 'cost',
|
|
|
+ 'addToCart', 'addToCartClicks', 'addToCartRate', 'brandedSearches',
|
|
|
+ 'brandedSearchesClicks', 'costType','detailPageViews', 'detailPageViewsClicks','eCPAddToCart',
|
|
|
+ 'newToBrandDetailPageViewRate', 'newToBrandDetailPageViews','newToBrandDetailPageViewsClicks','newToBrandECPDetailPageView',
|
|
|
+ 'newToBrandSales','newToBrandSalesClicks', 'newToBrandPurchases',
|
|
|
+ 'newToBrandPurchasesClicks', 'newToBrandPurchasesPercentage', 'newToBrandPurchasesRate', 'newToBrandUnitsSold', 'newToBrandUnitsSoldClicks',
|
|
|
+ 'newToBrandUnitsSoldPercentage','newToBrandSalesPercentage','date','purchases', 'purchasesClicks', 'purchasesPromoted', 'sales', 'salesClicks',
|
|
|
+ 'salesPromoted', 'unitsSold', 'unitsSoldClicks', 'video5SecondViewRate', 'video5SecondViews',
|
|
|
+ 'videoCompleteViews', 'videoFirstQuartileViews', 'videoMidpointViews', 'videoThirdQuartileViews', 'videoUnmutes',
|
|
|
+ 'viewabilityRate', 'viewableImpressions'
|
|
|
+ ] # 'startDate', 'endDate',
|
|
|
+ params['groupby'] = ['ads']
|
|
|
+ params['timeUnit'] = 'DAILY'
|
|
|
+ list_report = self.get_v3_report(timeUnit=params['timeUnit'], groupby=params['groupby'],
|
|
|
+ columns=params['columns'], startDate=params['startDate'],
|
|
|
+ endDate=params['endDate'], reportType=params['reportType'])
|
|
|
+ # print(list_report)
|
|
|
+ df_report = pd.json_normalize(list_report)
|
|
|
+ df_report['date'] = df_report['date'].map(lambda x: datetime.fromtimestamp(int(x)/1000).date(),na_action='ignore')#,na_action='ignore'
|
|
|
+ df_report = self.type_trans(df_report, params['columns'], timeZone_, extra_columns=[])
|
|
|
+
|
|
|
+ # print(df_report)
|
|
|
+ conn.insert_df("AmazonReport.SB_sbAds_adsV3", df_report[params['columns']])
|
|
|
+ time.sleep(0.05)
|
|
|
+ print("插入完成SB_sbAds_adsV3")
|
|
|
+
|
|
|
+ df_rel = df_report[params['columns']]
|
|
|
+ return df_rel
|
|
|
|
|
|
def reportV3_purchasedAsinRecord_ETL(self, conn, params:dict=None):
|
|
|
print(params)
|
|
@@ -1872,59 +2026,15 @@ if __name__ == '__main__':
|
|
|
}
|
|
|
conn = Common_ETLMethod(**AWS_CREDENTIALS).clickhouse_connect()
|
|
|
|
|
|
- # SD
|
|
|
- ac_etl = SP_ETL(**AWS_CREDENTIALS)
|
|
|
- # ls = ac_etl.reportV2_campaignsRecord_t2_ETL(conn)
|
|
|
- ls = ac_etl.targets_ETL()
|
|
|
- print(ls)
|
|
|
- # ls.to_csv('temp.csv')
|
|
|
- # print(ls)
|
|
|
- # print(ls.info())
|
|
|
- # print(ls.to_excel('obse11.xlsx'))
|
|
|
- # ac_etl.reportV2_campaignsRecord_t2_ETL(conn)
|
|
|
- # ac_etl.reportV2_campaignsRecord_t3_ETL(conn)
|
|
|
- # ac_etl.reportV2_adGroupsRecord_t2_ETL(conn)
|
|
|
- # ac_etl.reportV2_adGroupsRecord_t3_ETL(conn)
|
|
|
- # ac_etl.reportV2_asins_t2_ETL(conn)
|
|
|
- # ac_etl.reportV2_asins_t3_ETL(conn)
|
|
|
- # ac_etl.reportV2_productAds_t2_ETL(conn)
|
|
|
- # ac_etl.reportV2_productAds_t3_ETL(conn)
|
|
|
- # ac_etl.reportV2_targets_t2_ETL(conn)
|
|
|
- # ac_etl.reportV2_productAds_t3_ETL(conn)
|
|
|
- # ac_etl.reportV2_campaign_matchedTarget_t2_ETL(conn)
|
|
|
- # ac_etl.reportV2_campaign_matchedTarget_t3_ETL(conn)
|
|
|
- # ac_etl.reportV2_adGroups_matchedTarget_t2_ETL(conn)
|
|
|
- # ac_etl.reportV2_adGroups_matchedTarget_t3_ETL(conn)
|
|
|
- # ac_etl.reportV2_targets_matchedTarget_t2_ETL(conn)
|
|
|
- # ac_etl.reportV2_targets_matchedTarget_t3_ETL(conn)
|
|
|
-
|
|
|
- # SB
|
|
|
- # ac_etl = SB_ETL(**AWS_CREDENTIALS)
|
|
|
- # ac_etl.reportV3_purchasedAsinRecord_ETL(conn)
|
|
|
- # ac_etl.reportV2_campaignsRecord_ETL(conn)
|
|
|
- # ac_etl.reportV2_campaignsVideo_ETL(conn)
|
|
|
- # ac_etl.reportV2_adGroupsRecord_ETL(conn)
|
|
|
- # ac_etl.reportV2_adGroupsVideo_ETL(conn)
|
|
|
- # ac_etl.reportV2_adsRecord_ETL(conn)
|
|
|
- # ac_etl.reportV2_adsVideo_ETL(conn)
|
|
|
- # ac_etl.reportV2_keywordsRecord_ETL(conn)
|
|
|
- # ac_etl.reportV2_keywordsVideo_ETL(conn)
|
|
|
- # ac_etl.reportV2_placementRecord_ETL(conn)
|
|
|
- # ac_etl.reportV2_placementVideo_ETL(conn)
|
|
|
- # ac_etl.reportV2_searchtermsRecord_ETL(conn)
|
|
|
- # ac_etl.reportV2_searchtermsVideo_ETL(conn)
|
|
|
- # ac_etl.reportV2_targetsRecord_ETL(conn)
|
|
|
- # ac_etl.reportV2_targetsVideo_ETL(conn)
|
|
|
-
|
|
|
- # SP
|
|
|
- # ac_etl = SP_ETL(**AWS_CREDENTIALS)
|
|
|
- # ac_etl.reportV3_campaign_spCampaignsETL(conn)
|
|
|
- # ac_etl.reportV3_adGroup_spCampaignsETL(conn)
|
|
|
- # ac_etl.reportV3_campaignPlacement_spCampaignsETL(conn)
|
|
|
- # ac_etl.reportV3_targeting_spTargetingETL(conn)
|
|
|
- # ac_etl.reportV3_searchTerm_spSearchTermETL(conn)
|
|
|
- # ac_etl.reportV3_advertiser_spAdvertisedProductETL(conn)
|
|
|
- # ac_etl.reportV3_asin_spPurchasedProductETL(conn)
|
|
|
-
|
|
|
- conn.close()
|
|
|
- ###
|
|
|
+ sb_ = SB_ETL(**AWS_CREDENTIALS)
|
|
|
+
|
|
|
+ # list_date = ['2023-11-20']
|
|
|
+ list_date = [f'2023-11-{"0" + str(i) if len(str(i)) == 1 else i}' for i in range(1, 22)]
|
|
|
+ for date_ in list_date:
|
|
|
+ print(date_)
|
|
|
+ print(date_.replace("-",""))
|
|
|
+ sb_ = SB_ETL(**AWS_CREDENTIALS)
|
|
|
+ rel = sb_.reportV3_sbAds_ETL(conn,params={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
|
|
|
+ print(rel)
|
|
|
+ print(rel.info())
|
|
|
+ print(rel.columns)
|