Merge branch 'master' into yifan

sync_amz_data/public/amz_ad_client.py

@@ -668,6 +668,21 @@ class SBClient(BaseClient):
         url_path = "/sb/targets/list"
         return self._request(url_path, method="POST", body=body)
 
+    def get_sbthemetargeting(self, **body):
+        url_path = "/sb/themes/list"
+        return self._request(url_path, method="POST", body=body)
+
+    def iter_sbthemetargeting(self, **body):
+        if "maxResults" not in body:
+            body["maxResults"] = 100
+        while True:
+            info: dict = self.get_sbthemetargeting(**body)
+            # print(info)
+            yield from info["themes"]
+            if not info.get("nextToken"):
+                break
+            body["nextToken"] = info["nextToken"]
+
     def iter_targets(self, **body):
         if "maxResults" not in body:
             body["maxResults"] = 100

sync_amz_data/tasks/datainsert/SB/mysql_datainsert_sbads.py

@@ -70,7 +70,7 @@ class SbAds:
         df['extendedData.lastUpdateDate'] = pd.to_datetime(df['extendedData.lastUpdateDate'],
                                                                unit='ms').dt.strftime('%Y-%m-%d %H:%M:%S')
         col = ['adId', 'campaignId', 'landingPage.asins', 'landingPage.pageType', 'landingPage.url', 'name', 'state',
-               'adGroupId', 'creative.asins', 'creative.type', 'extendedData.creationDate',
+               'adGroupId', 'creative.asins', 'creative.type', 'creative.headline', 'extendedData.creationDate',
                'extendedData.lastUpdateDate', 'extendedData.servingStatus', 'extendedData.servingStatusDetails']
         for i in col:
             if i not in df.columns:

sync_amz_data/tasks/datainsert/SB/mysql_datainsert_sbgroup.py

@@ -134,7 +134,7 @@ class SbGroup:
                                           }})
         df_group = pd.json_normalize(list(list_group))
         return df_group
-    #----------------------------
+    # ----------------------------
 
     def dataconvert(self):
         df = self.get_spgroup_data()

sync_amz_data/tasks/datainsert/SB/mysql_datainsert_sbthemetargeting_v3.py

@@ -0,0 +1,87 @@
+from sync_amz_data.DataTransform.Data_ETL import SB_ETL
+import requests
+from urllib.parse import urljoin
+from sync_amz_data.settings import AWS_LWA_CLIENT
+import pandas as pd
+import json
+from sync_amz_data.tasks.datainsert.wg import LADS
+from sync_amz_data.public.amz_ad_client import SBClient
+from typing import Literal
+
+
+class RateLimitError(Exception):
+    def __init__(self, retry_after: str = None):
+        self.retry_after = retry_after
+
+
+def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None, AD=LADS):
+    ADS = AD
+    resp = requests.session().request(
+        method=method,
+        url=urljoin(ADS, url_path),
+        headers=head,
+        params=params,
+        json=body,
+    )
+    if resp.status_code == 429:
+        raise RateLimitError(resp.headers.get("Retry-After"))
+    if resp.status_code >= 400:
+        raise Exception(resp.text)
+    return resp.json()
+
+
+class SbThemeTargeting:
+    def __init__(self, profile_id):
+        self.profile_id = profile_id
+        self.re_url_path = "api/ad_manage/profiles/"
+        self.upcreate_url_path = "api/ad_manage/sbthemetargeting/p/updata/"
+        self.heads = {'X-Token': "da4ab6bc5cbf1dfa"}
+        self.refresh_token = self.get_refresh_token()
+        self.lwa_client_id = AWS_LWA_CLIENT['lwa_client_id']
+        self.lwa_client_secret = AWS_LWA_CLIENT['lwa_client_secret']
+        self.AWS_CREDENTIALS = {
+            'lwa_client_id': self.lwa_client_id,
+            'lwa_client_secret': self.lwa_client_secret,
+            'refresh_token': self.refresh_token,
+            'profile_id': self.profile_id
+        }
+
+    def get_refresh_token(self):
+        params = {'profile_id': self.profile_id}
+        heads = self.heads
+        url_path = self.re_url_path
+        tem = request(url_path=url_path, head=heads, params=params)
+        if tem.get('data') is not None:
+            _ = tem.get('data')
+            out = _[0].get('refresh_token')
+        else:
+            out = None
+        return out
+
+    def get_sbthemetargeting_data(self):
+        tem = SBClient(**self.AWS_CREDENTIALS)
+        list_ads = tem.iter_sbthemetargeting(**{"maxResults": 5000})
+        df_ads = pd.json_normalize(list(list_ads))
+
+        df_ads.rename(columns={
+            'adGroupId': 'adGroup',
+            'campaignId': 'campaign',
+        }, inplace=True)
+        df_ads['profile'] = self.profile_id
+        return df_ads
+
+    def updata_create(self):
+        df_data = self.get_sbthemetargeting_data()
+        _ = df_data.to_json(orient='records', date_format='iso')
+        body = json.loads(_)
+        heads = self.heads
+        url_path = self.upcreate_url_path
+        tem = request(url_path=url_path, head=heads, body=body, method="POST")
+        return tem
+
+
+if __name__ == '__main__':
+    a = SbThemeTargeting(profile_id="3006125408623189")
+    # out = a.get_sbthemetargeting_data()
+    out = a.updata_create()
+    print(out)

sync_amz_data/tasks/datainsert/alldata_insert.py

@@ -18,6 +18,9 @@ from SB.mysql_datainsert_sb_keywordsbid_recommendations_v3 import SbkeywordsBidR
 from SB.mysql_datainsert_sbtarget_v3 import SbTargets
 from SB.mysql_datainsert_sbtargetbid_recommendations_v3 import SbtargetsBidRecommendations
 from SB.mysql_datainsert_sbnegativekeyword import SbNegtiveKeyword
+from SB.mysql_datainsert_sbthemetargeting_v3 import SbThemeTargeting
+
+from mysql_datainsert_assets import Assets
 
 import time
 
@@ -145,3 +148,15 @@ Sbnk = SbNegtiveKeyword(profile_id="3006125408623189")
 Sbnko = Sbnk.updata_create()
 print("SbNegtiveKeyword", Sbnko)
 protime(start_time)
+
+start_time = time.time()
+Sbtt = SbThemeTargeting(profile_id="3006125408623189")
+Sbtto = Sbtt.updata_create()
+print("SbThemeTargeting", Sbtto)
+protime(start_time)
+
+start_time = time.time()
+ass = Assets(profile_id="3006125408623189")
+asso = ass.updata_create()
+print("Assets", asso)
+protime(start_time)

sync_amz_data/tasks/datainsert/mysql_datainsert_assets.py

@@ -0,0 +1,109 @@
+import requests
+from urllib.parse import urljoin
+from sync_amz_data.public.amz_ad_client import BaseClient
+from sync_amz_data.settings import AWS_LWA_CLIENT
+import pandas as pd
+import json
+from sync_amz_data.tasks.datainsert.wg import LADS
+
+
+class Client(BaseClient):
+    def assets_search(self, **body):
+        url_path = "/assets/search"
+        headers = {
+            "Accept": "application/vnd.creativeassetsgetresponse.v3+json",
+            "Content-Type": "application/vnd.creativeassetsgetresponse.v3+json"
+        }
+        return self._request(url_path, method="POST", headers=headers, body=body)
+
+
+class RateLimitError(Exception):
+    def __init__(self, retry_after: str = None):
+        self.retry_after = retry_after
+
+
+def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None, AD = LADS):
+    ADS = AD
+    resp = requests.session().request(
+        method=method,
+        url=urljoin(ADS, url_path),
+        headers=head,
+        params=params,
+        json=body,
+    )
+    if resp.status_code == 429:
+        raise RateLimitError(resp.headers.get("Retry-After"))
+    if resp.status_code >= 400:
+        raise Exception(resp.text)
+    return resp.json()
+
+
+class Assets:
+    def __init__(self, profile_id):
+        self.profile_id = profile_id
+        self.re_url_path = "api/ad_manage/profiles/"
+        self.upcreate_url_path = "api/ad_manage/sb/assets/p/updata/"
+        self.heads = {'X-Token': "da4ab6bc5cbf1dfa"}
+        self.refresh_token = self.get_refresh_token()
+        self.lwa_client_id = AWS_LWA_CLIENT['lwa_client_id']
+        self.lwa_client_secret = AWS_LWA_CLIENT['lwa_client_secret']
+        self.AWS_CREDENTIALS = {
+            'lwa_client_id': self.lwa_client_id,
+            'lwa_client_secret': self.lwa_client_secret,
+            'refresh_token': self.refresh_token,
+            'profile_id': self.profile_id
+        }
+
+    def get_refresh_token(self):
+        params = {'profile_id': self.profile_id}
+        heads = self.heads
+        url_path = self.re_url_path
+        tem = request(url_path=url_path, head=heads, params=params)
+        if tem.get('data') is not None:
+            _ = tem.get('data')
+            out = _[0].get('refresh_token')
+        else:
+            out = None
+        return out
+
+    def get_assets_data(self):
+        tem = Client(**self.AWS_CREDENTIALS)
+        tokentext = ""
+        assetList = []
+        while tokentext != None:
+            body = {
+                    "pageCriteria": {
+                        "identifier": {
+                            "token": tokentext
+                            },
+                        "size": 500
+                        }
+                    }
+            list_assets = tem.assets_search(**body)
+            tokentext = list_assets.get('token', None)
+            assetList.extend(list_assets.get('assetList', []))
+        df_assets = pd.DataFrame(assetList)
+        df_assets['profile_id'] = self.profile_id
+        return df_assets
+
+    def dataconvert(self):
+        df = self.get_assets_data()
+        df['creationTime'] = pd.to_datetime(df['creationTime'], unit='ms').dt.strftime('%Y-%m-%d')
+        df['lastUpdatedTime'] = pd.to_datetime(df['lastUpdatedTime'], unit='ms').dt.strftime('%Y-%m-%d')
+        df['assetSubTypeList'] = df['assetSubTypeList'].astype(str)
+        json_data = json.loads(df.to_json(orient='records', force_ascii=False))
+        return json_data
+
+    def updata_create(self):
+        body = self.dataconvert()
+        heads = self.heads
+        url_path = self.upcreate_url_path
+        tem = request(url_path=url_path, head=heads, body=body, method="POST")
+        return tem
+
+
+if __name__ == '__main__':
+    a = Assets(profile_id="3006125408623189")
+    # out = a.dataconvert()
+    out = a.updata_create()
+    print(out)

sync_amz_data/tasks/datainsert/wg.py

@@ -1 +1 @@
-LADS = "http://192.168.1.39:8001/"
+LADS = "http://192.168.1.18:8001/"