Merge branch 'wengao' of ASJ_ADS/sync_amz_data into master

sync_amz_data/DataTransform/Data_ETL.py

@@ -171,12 +171,15 @@ class SP_ETL(SPClient, Common_ETLMethod):
     def targets_ETL(self):
         list_targets = list(self.iter_targets(**{"includeExtendedDataFields": True}))
         df_targets = pd.json_normalize(list_targets)
-        print("--")
         df_targets = self.TZ_Deal(df_targets, ["extendedData.creationDateTime", "extendedData.lastUpdateDateTime"])
         df_targets['resolvedExpressions_type'] = df_targets['resolvedExpression'].map(
             lambda x: self.get_keyOvalue(x, 'type'))
         df_targets['resolvedExpressions_value'] = df_targets['resolvedExpression'].map(
             lambda x: self.get_keyOvalue(x, 'value'))
+        df_targets['expression_type'] = df_targets['expression'].map(
+            lambda x: self.get_keyOvalue(x, 'type'))
+        df_targets['expression_value'] = df_targets['expression'].map(
+            lambda x: self.get_keyOvalue(x, 'value'))
         return self.columnsName_modify(df_targets)
 
     def budget_ETL(self, campaign_ids: list):

sync_amz_data/tasks/datainsert/SP/mysql_datainsert_sptarget.py

@@ -0,0 +1,97 @@
+from sync_amz_data.DataTransform.Data_ETL import SP_ETL
+
+import requests
+from urllib.parse import urljoin
+from sync_amz_data.public.amz_ad_client import SPClient
+from sync_amz_data.settings import AWS_LWA_CLIENT
+import pandas as pd
+import json
+
+
+class RateLimitError(Exception):
+    def __init__(self, retry_after: str = None):
+        self.retry_after = retry_after
+
+
+def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None):
+    ADS = "http://192.168.1.23:8001/"
+    resp = requests.session().request(
+        method=method,
+        url=urljoin(ADS, url_path),
+        headers=head,
+        params=params,
+        json=body,
+    )
+    if resp.status_code == 429:
+        raise RateLimitError(resp.headers.get("Retry-After"))
+    if resp.status_code >= 400:
+        raise Exception(resp.text)
+    return resp.json()
+
+
+class SpTargets:
+    def __init__(self, profile_id):
+        self.profile_id = profile_id
+        self.re_url_path = "api/ad_manage/profiles/"
+        self.upcreate_url_path = "api/ad_manage/sptargets/updata/?updata=1"
+        self.heads = {'X-Token': "da4ab6bc5cbf1dfa"}
+        self.refresh_token = self.get_refresh_token()
+        self.lwa_client_id = AWS_LWA_CLIENT['lwa_client_id']
+        self.lwa_client_secret = AWS_LWA_CLIENT['lwa_client_secret']
+        self.AWS_CREDENTIALS = {
+            'lwa_client_id': self.lwa_client_id,
+            'lwa_client_secret': self.lwa_client_secret,
+            'refresh_token': self.refresh_token,
+            'profile_id': self.profile_id
+        }
+
+    def get_refresh_token(self):
+        params = {'profile_id': self.profile_id}
+        heads = self.heads
+        url_path = self.re_url_path
+        tem = request(url_path=url_path, head=heads, params=params)
+        if tem.get('data') is not None:
+            _ = tem.get('data')
+            out = _[0].get('refresh_token')
+        else:
+            out = None
+        return out
+
+
+    def get_sptargets_data(self):
+        tem = SP_ETL(**self.AWS_CREDENTIALS)
+        df = tem.targets_ETL()
+        if len(df) > 0:
+            df.rename(columns={'adgroupid': 'adGroup',
+                               'campaignid': 'campaign',
+                               'expressiontype': 'expressionType',
+                               'targetid': 'targetId',
+                               'extendeddata_creationdatetime': 'creationDateTime',
+                               'extendeddata_lastupdatedatetime': 'lastUpdateDateTime',
+                               'extendeddata_servingstatus': 'servingStatus',
+                               'extendeddata_servingstatusdetails': 'servingStatusDetails',
+                               'resolvedexpressions_type': 'resolvedExpression_type',
+                               'resolvedexpressions_value': 'resolvedExpression_value'}, inplace=True)
+            df.drop(columns=['expression', 'resolvedexpression'], inplace=True)
+            df['profile'] = self.profile_id
+            df['creationDateTime'] = pd.to_datetime(df['creationDateTime']).dt.strftime(
+                '%Y-%m-%d %H:%M:%S')
+            df['lastUpdateDateTime'] = pd.to_datetime(df['lastUpdateDateTime']).dt.strftime(
+                '%Y-%m-%d %H:%M:%S')
+        return df
+
+    def updata_create(self):
+        df_data = self.get_sptargets_data()
+        _ = df_data.to_json(orient='records', date_format='iso')
+        body = json.loads(_)
+        heads = self.heads
+        url_path = self.upcreate_url_path
+        tem = request(url_path=url_path, head=heads, body=body, method="POST")
+        return tem
+
+
+if __name__ == '__main__':
+    a = SpTargets(profile_id="3006125408623189")
+    # out = a.get_sptargets_data()
+    out = a.updata_create()
+    print(out)

sync_amz_data/tasks/datainsert/alldata_insert.py

@@ -5,6 +5,7 @@ from SP.mysql_datainsert_spkeyword import SpKeyword
 from SP.mysql_datainsert_sp_budget_recommendation import SpBudgetRecommendation
 from SP.mysql_datainsert_spads import SpAds
 from SP.mysql_datainsert_sp_targetsbid_recommendations import SpTargetsBidRecommendations
+from SP.mysql_datainsert_sptarget import SpTargets
 
 
 pf = Portfolios("3006125408623189")
@@ -34,3 +35,7 @@ print("SpBudgetRecommendation", spbro)
 sptb = SpTargetsBidRecommendations(profile_id="3006125408623189")
 sptbo = sptb.updata_create()
 print("SpTargetsBidRecommendations", sptbo)
+
+spt = SpTargets(profile_id="3006125408623189")
+spto = spt.updata_create()
+print("SpTargets", spto)