Merge branch 'wengao' of ASJ_ADS/sync_amz_data into master

sync_amz_data/tasks/datainsert/SP/mysql_datainsert_sp_targetsbid_recommendations.py

@@ -127,6 +127,6 @@ class SpTargetsBidRecommendations:
 
 if __name__ == '__main__':
     a = SpTargetsBidRecommendations(profile_id="3006125408623189")
-    # out = a.get_sptargetsbidrecommendation_data()
-    out = a.updata_create()
+    out = a.get_sptargetsbidrecommendation_data()
+    # out = a.updata_create()
     print(out)

sync_amz_data/tasks/datainsert/SP/mysql_datainsert_sp_targetsbid_recommendations_v2.py

@@ -0,0 +1,140 @@
+import requests
+from urllib.parse import urljoin
+from sync_amz_data.public.amz_ad_client import SPClient
+from sync_amz_data.settings import AWS_LWA_CLIENT
+import pandas as pd
+import json
+
+
+class RateLimitError(Exception):
+    def __init__(self, retry_after: str = None):
+        self.retry_after = retry_after
+
+
+def convert_row(row):
+    type_list = row['type']
+    value_list = row['value']
+    expressions = []
+    if isinstance(row['value'], list):
+        for i in range(len(value_list)):
+            for v, t in zip(value_list[i:i+1], type_list[i:i+1]):
+                v = v.replace("'", "")
+                v = v.replace(" ", "")
+                t = t.replace("'", "")
+                t = t.replace(" ", "")
+                temp_dict = {"type": str(t), "value": str(v)}
+                expressions.append(temp_dict)
+    else:
+        expressions = [{"type": type_list, "value": value_list}]
+    return expressions
+
+def convert_list(val):
+    if '[' in val:
+        return val.split('[')[1].split(']')[0].split(',')
+    else:
+        return val
+
+def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None):
+    ADS = "http://192.168.1.23:8001/"
+    resp = requests.session().request(
+        method=method,
+        url=urljoin(ADS, url_path),
+        headers=head,
+        params=params,
+        json=body,
+    )
+    if resp.status_code == 429:
+        raise RateLimitError(resp.headers.get("Retry-After"))
+    if resp.status_code >= 400:
+        raise Exception(resp.text)
+    return resp.json()
+
+
+class SpTargetsBidRecommendationsV2:
+    def __init__(self, profile_id):
+        self.profile_id = profile_id
+        self.re_url_path = "api/ad_manage/profiles/"
+        self.spt_url_path = "api/ad_manage/sptargetsv2/"
+        self.upcreate_url_path = "api/ad_manage/sptargetsbidrecommendationv2/updata/"
+        self.heads = {'X-Token': "da4ab6bc5cbf1dfa"}
+        self.refresh_token = self.get_refresh_token()
+        self.lwa_client_id = AWS_LWA_CLIENT['lwa_client_id']
+        self.lwa_client_secret = AWS_LWA_CLIENT['lwa_client_secret']
+        self.AWS_CREDENTIALS = {
+            'lwa_client_id': self.lwa_client_id,
+            'lwa_client_secret': self.lwa_client_secret,
+            'refresh_token': self.refresh_token,
+            'profile_id': self.profile_id
+        }
+
+    def get_refresh_token(self):
+        params = {'profile_id': self.profile_id}
+        heads = self.heads
+        url_path = self.re_url_path
+        tem = request(url_path=url_path, head=heads, params=params)
+        if tem.get('data') is not None:
+            _ = tem.get('data')
+            out = _[0].get('refresh_token')
+        else:
+            out = None
+        return out
+
+    def get_arg(self):
+        heads = self.heads
+        url_path = self.spt_url_path
+        data = []
+        page = 1
+        params = {'profile_id': self.profile_id, 'limit': 999, 'page': page}
+        tem = request(url_path=url_path, head=heads, params=params)
+        data.extend(tem.get('data'))
+        while tem.get('is_next') is True:
+            page += 1
+            params = {'profile_id': self.profile_id, 'limit': 999, 'page': page}
+            tem = request(url_path=url_path, head=heads, params=params)
+            data.extend(tem.get('data'))
+        _ = pd.json_normalize(data)
+        df = _.copy()
+        df['expression_type'] = df['expression_type'].str.replace('_', '')
+        df.rename(columns={'expression_value': 'value', 'expression_type': 'type'}, inplace=True)
+        df['value'] = df['value'].apply(convert_list)
+        df['type'] = df['type'].apply(convert_list)
+        df['expressions'] = df.apply(convert_row, axis=1)
+        df_grouped = df.groupby('adGroupId').agg({'expressions': list}).reset_index()
+        return df, df_grouped
+
+    def get_sptargetsbidrecommendation_data(self):
+        tem = SPClient(**self.AWS_CREDENTIALS)
+        df_old, df_arg = self.get_arg()
+        data_json = df_arg.to_json(orient='records')
+        list_arg = json.loads(data_json)
+        out_data = []
+        for i in list_arg:
+            res = tem.iter_bidrecommendationList(**i)
+            out_data.extend(list(res))
+        temtest = pd.json_normalize(out_data)
+        temtest['expression'] = temtest['expression'].astype(str).str.lower()
+        df_old.rename(columns={'expressions': 'expression'}, inplace=True)
+        df_old['expression'] = df_old['expression'].astype(str).str.lower()
+        _ = pd.merge(left=df_old, right=temtest, on=['expression', 'adGroupId'], how='left')
+        outall = _[['targetId', 'suggestedBid.rangeEnd', 'suggestedBid.rangeStart', 'suggestedBid.suggested']].copy()
+        outall = outall.dropna(subset=['suggestedBid.rangeEnd'])
+        outall.rename(columns={'suggestedBid.suggested': 'suggestedBid',
+                               'suggestedBid.rangeStart': 'suggestedBid_lower',
+                               'suggestedBid.rangeEnd': 'suggestedBid_upper',
+                               'targetId': 'target'
+                               }, inplace=True)
+        json_data = json.loads(outall.to_json(orient='records', force_ascii=False))
+        return json_data
+
+    def updata_create(self):
+        body = self.get_sptargetsbidrecommendation_data()
+        heads = self.heads
+        url_path = self.upcreate_url_path
+        tem = request(url_path=url_path, head=heads, body=body, method="POST")
+        return tem
+
+if __name__ == '__main__':
+    a = SpTargetsBidRecommendationsV2(profile_id="3006125408623189")
+    # out = a.get_sptargetsbidrecommendation_data()
+    out = a.updata_create()
+    print(out)