Merge remote-tracking branch 'origin/master' into yifan

sync_amz_data/public/amz_ad_client.py

@@ -261,7 +261,7 @@ class SPClient(BaseClient):
                 info = self.get_adgroup_bidrecommendation(campaignId,adGroupId,targetingExpressions[i:i+100],recommendationType)
                 yield from info['bidRecommendations']
             except:
-                print("空值")
+                # print("空值")
                 return iter([])
 
     def get_keyword_bidrecommendation(self, adGroupId: str, keyword: list, matchType: list):

sync_amz_data/tasks/datainsert/SP/mysql_datainsert_sp_targetsbid_recommendations.py

@@ -0,0 +1,132 @@
+import requests
+from urllib.parse import urljoin
+from sync_amz_data.public.amz_ad_client import SPClient
+from sync_amz_data.settings import AWS_LWA_CLIENT
+import pandas as pd
+import json
+
+pd.set_option('display.max_columns', None)
+# 显示所有行
+pd.set_option('display.max_rows', None)
+
+
+class RateLimitError(Exception):
+    def __init__(self, retry_after: str = None):
+        self.retry_after = retry_after
+
+
+def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None):
+    ADS = "http://192.168.1.23:8001/"
+    resp = requests.session().request(
+        method=method,
+        url=urljoin(ADS, url_path),
+        headers=head,
+        params=params,
+        json=body,
+    )
+    if resp.status_code == 429:
+        raise RateLimitError(resp.headers.get("Retry-After"))
+    if resp.status_code >= 400:
+        raise Exception(resp.text)
+    return resp.json()
+
+
+class SpTargetsBidRecommendations:
+    def __init__(self, profile_id):
+        self.profile_id = profile_id
+        self.re_url_path = "api/ad_manage/profiles/"
+        self.cgk_url_path = "api/ad_manage/sptbrkeywords/"
+        self.upcreate_url_path = "api/ad_manage/sptargetsbidrecommendation/updata/"
+        self.heads = {'X-Token': "da4ab6bc5cbf1dfa"}
+        self.refresh_token = self.get_refresh_token()
+        self.lwa_client_id = AWS_LWA_CLIENT['lwa_client_id']
+        self.lwa_client_secret = AWS_LWA_CLIENT['lwa_client_secret']
+        self.AWS_CREDENTIALS = {
+            'lwa_client_id': self.lwa_client_id,
+            'lwa_client_secret': self.lwa_client_secret,
+            'refresh_token': self.refresh_token,
+            'profile_id': self.profile_id
+        }
+
+    def get_refresh_token(self):
+        params = {'profile_id': self.profile_id}
+        heads = self.heads
+        url_path = self.re_url_path
+        tem = request(url_path=url_path, head=heads, params=params)
+        if tem.get('data') is not None:
+            _ = tem.get('data')
+            out = _[0].get('refresh_token')
+        else:
+            out = None
+        return out
+
+    def get_arg(self):
+        heads = self.heads
+        url_path = self.cgk_url_path
+        data = []
+        page = 1
+        params = {'profile_id': self.profile_id, 'limit': 999, 'page': page}
+        tem = request(url_path=url_path, head=heads, params=params)
+        data.extend(tem.get('data'))
+        while tem.get('is_next') is True:
+            page += 1
+            params = {'profile_id': self.profile_id, 'limit': 999, 'page': page}
+            tem = request(url_path=url_path, head=heads, params=params)
+            data.extend(tem.get('data'))
+        _ = pd.json_normalize(data)
+        df = _.copy()
+        df.rename(columns={'keywordText': 'value', 'matchType': 'type'}, inplace=True)
+        df['targetingExpressions'] = df[['value', 'type', 'keywordId']].apply(lambda x: x.to_dict(), axis=1)
+        df_grouped = df.groupby(['campaignId', 'adGroupId']).agg({'targetingExpressions': list}).reset_index()
+        return df_grouped
+
+    def get_sptargetsbidrecommendation_data(self):
+        tem = SPClient(**self.AWS_CREDENTIALS)
+        df_arg = self.get_arg()
+        data_json = df_arg.to_json(orient='records')
+        list_arg = json.loads(data_json)
+        out_df = pd.DataFrame()
+        for i in list_arg:
+            k_id_text_df = pd.DataFrame.from_dict(i['targetingExpressions'])
+            list_sptbr = tem.iter_adgroup_bidrecommendation(**i)
+            list_outdata = list(list_sptbr)
+            if len(list_outdata) > 0:
+                out_data = []
+                for j in list_outdata:
+                    if j.get('theme') == "CONVERSION_OPPORTUNITIES":
+                        data = j.get('bidRecommendationsForTargetingExpressions')
+                        out_data.extend(data)
+                temtest = pd.json_normalize(out_data)
+                temtest.rename(columns={'targetingExpression.value': 'value'}, inplace=True)
+                temtest.rename(columns={'targetingExpression.type': 'type'}, inplace=True)
+                df_tem = pd.merge(left=temtest, right=k_id_text_df, on=['value', 'type'], how='left')
+                out_df = pd.concat([out_df, df_tem])
+        out_df['suggestedBid'] = out_df.apply(
+            lambda row: row['bidValues'][1]['suggestedBid'] if len(row['bidValues']) > 0 else None,
+            axis=1)
+        out_df['suggestedBid_lower'] = out_df.apply(
+            lambda row: row['bidValues'][0]['suggestedBid'] if len(row['bidValues']) > 0 else None,
+            axis=1)
+        out_df['suggestedBid_upper'] = out_df.apply(
+            lambda row: row['bidValues'][2]['suggestedBid'] if len(row['bidValues']) > 0 else None,
+            axis=1)
+        out_df.drop(labels='bidValues', inplace=True, axis=1)
+        out_df.drop(labels='value', inplace=True, axis=1)
+        out_df.rename(columns={'type': 'targetingExpression_type',
+                               'keywordId': 'keyword'}, inplace=True)
+        json_data = json.loads(out_df.to_json(orient='records', force_ascii=False))
+        return json_data
+
+    def updata_create(self):
+        body = self.get_sptargetsbidrecommendation_data()
+        heads = self.heads
+        url_path = self.upcreate_url_path
+        tem = request(url_path=url_path, head=heads, body=body, method="POST")
+        return tem
+
+
+if __name__ == '__main__':
+    a = SpTargetsBidRecommendations(profile_id="3006125408623189")
+    # out = a.get_sptargetsbidrecommendation_data()
+    out = a.updata_create()
+    print(out)

sync_amz_data/tasks/datainsert/SP/mysql_datainsert_spkeyword.py

@@ -30,9 +30,9 @@ def request(url_path: str, method: str = "GET", head: dict = None, params: dict
 class SpKeyword:
     def __init__(self, profile_id, campaignId: list = None):
         self.profile_id = profile_id
-        self.portfolioId = campaignId
+        self.campaignId = campaignId
         self.re_url_path = "api/ad_manage/profiles/"
-        self.upcreate_url_path = "api/ad_manage/spkeywords/updata/"
+        self.upcreate_url_path = "api/ad_manage/spkeywords/updata/?updata=1"
         self.heads = {'X-Token': "da4ab6bc5cbf1dfa"}
         self.refresh_token = self.get_refresh_token()
         self.lwa_client_id = AWS_LWA_CLIENT['lwa_client_id']
@@ -99,4 +99,4 @@ if __name__ == '__main__':
     a = SpKeyword(profile_id="3006125408623189")
     out = a.updata_create()
     # out = a.dataconvert()
-    print(out)
+    print(out)

sync_amz_data/tasks/datainsert/alldata_insert.py

@@ -4,28 +4,33 @@ from SP.mysql_datainsert_spgroup import SpGroup
 from SP.mysql_datainsert_spkeyword import SpKeyword
 from SP.mysql_datainsert_sp_budget_recommendation import SpBudgetRecommendation
 from SP.mysql_datainsert_spads import SpAds
+from SP.mysql_datainsert_sp_targetsbid_recommendations import SpTargetsBidRecommendations
 
 
 pf = Portfolios("3006125408623189")
 pfo = pf.updata_create()
 print("Portfolios", pfo)
 
-spc = SpCampaign("3006125408623189")
+spc = SpCampaign(profile_id="3006125408623189")
 spco = spc.updata_create()
 print("SpCampaign", spco)
 
-spg = SpGroup("3006125408623189")
+spg = SpGroup(profile_id="3006125408623189")
 spgo = spg.updata_create()
 print("SpGroup", spgo)
 
-spk = SpKeyword("3006125408623189")
+spk = SpKeyword(profile_id="3006125408623189")
 spko = spk.updata_create()
 print("SpKeyword", spko)
 
-spa = SpAds("3006125408623189")
+spa = SpAds(profile_id="3006125408623189")
 spao = spa.updata_create()
 print("SpAds", spao)
 
-spbr = SpBudgetRecommendation("3006125408623189")
+spbr = SpBudgetRecommendation(profile_id="3006125408623189")
 spbro = spbr.updata_create()
 print("SpBudgetRecommendation", spbro)
+
+sptb = SpTargetsBidRecommendations(profile_id="3006125408623189")
+sptbo = sptb.updata_create()
+print("SpTargetsBidRecommendations", sptbo)