Merge branch 'wengao' of ASJ_ADS/sync_amz_data into master

sync_amz_data/public/amz_ad_client.py

@@ -709,6 +709,13 @@ class SBClient(BaseClient):
                 # print("空值")
                 return iter([])
 
+    def get_negative_keyword(self, **param):
+        url_path = "/sb/negativeKeywords"
+        headers = {
+            "Accept": "application/vnd.sbnegativekeyword.v3.2+json",
+            "Content-Type": "application/json"
+        }
+        return self._request(url_path, method="GET", headers=headers, params=param)
 
     def get_v3_report(self,
                       groupby:list,

sync_amz_data/tasks/__init__.py

@@ -1,2 +1,2 @@
-from .sb import SBTask
+# from .sb import SBTask
 

sync_amz_data/tasks/datainsert/SB/mysql_datainsert_sb_keywordsbid_recommendations_v3.py

@@ -4,6 +4,7 @@ from sync_amz_data.public.amz_ad_client import SBClient
 from sync_amz_data.settings import AWS_LWA_CLIENT
 import pandas as pd
 import json
+from sync_amz_data.tasks.datainsert.wg import LADS
 
 pd.set_option('display.max_columns', None)
 # 显示所有行
@@ -15,8 +16,9 @@ class RateLimitError(Exception):
         self.retry_after = retry_after
 
 
-def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None):
-    ADS = "http://192.168.1.23:8001/"
+
+def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None, AD = LADS):
+    ADS = AD
     resp = requests.session().request(
         method=method,
         url=urljoin(ADS, url_path),

sync_amz_data/tasks/datainsert/SB/mysql_datainsert_sbads.py

@@ -4,6 +4,7 @@ from sync_amz_data.public.amz_ad_client import SBClient
 from sync_amz_data.settings import AWS_LWA_CLIENT
 import pandas as pd
 import json
+from sync_amz_data.tasks.datainsert.wg import LADS
 
 
 class RateLimitError(Exception):
@@ -11,8 +12,8 @@ class RateLimitError(Exception):
         self.retry_after = retry_after
 
 
-def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None):
-    ADS = "http://192.168.1.23:8001/"
+def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None, AD = LADS):
+    ADS = AD
     resp = requests.session().request(
         method=method,
         url=urljoin(ADS, url_path),

sync_amz_data/tasks/datainsert/SB/mysql_datainsert_sbcampaign.py

@@ -4,6 +4,7 @@ from sync_amz_data.public.amz_ad_client import SBClient
 from sync_amz_data.settings import AWS_LWA_CLIENT
 import pandas as pd
 import json
+from sync_amz_data.tasks.datainsert.wg import LADS
 
 
 pd.set_option('display.max_columns', None)
@@ -15,6 +16,7 @@ def convert_row(row):
     l_d = None
     l_o = None
     l_h = None
+    l_t = None
     if len(row['dyp']) > 0:
         for i in row['dyp']:
             a = json.loads(i)
@@ -25,7 +27,9 @@ def convert_row(row):
                     l_o = a['percentage']
                 if a['placement'] == "HOME":
                     l_h = a['percentage']
-    return l_d, l_o, l_h
+                if a['placement'] == "top":
+                    l_h = a['percentage']
+    return l_d, l_o, l_h, l_t
 
 def parse_dict_list(lst):
     if lst == 999:
@@ -38,8 +42,8 @@ class RateLimitError(Exception):
         self.retry_after = retry_after
 
 
-def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None):
-    ADS = "http://192.168.1.23:8001/"
+def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None, AD = LADS):
+    ADS = AD
     resp = requests.session().request(
         method=method,
         url=urljoin(ADS, url_path),
@@ -119,7 +123,7 @@ class SbCampaign:
         tem.columns = [i.replace(".", "_") for i in tem.columns]
         tem.bidding_bidAdjustmentsByPlacement = tem.bidding_bidAdjustmentsByPlacement.fillna(999)
         tem['dyp'] = tem.bidding_bidAdjustmentsByPlacement.apply(parse_dict_list)
-        tem[['detatil_page', 'other', 'home']] = tem.apply(convert_row, axis=1, result_type='expand')
+        tem[['detatil_page', 'other', 'home', 'top']] = tem.apply(convert_row, axis=1, result_type='expand')
         tem.drop(columns='dyp', inplace=True)
         tem.rename(columns={
             'name': 'campaignName',

sync_amz_data/tasks/datainsert/SB/mysql_datainsert_sbgroup.py

@@ -4,6 +4,7 @@ from sync_amz_data.public.amz_ad_client import SBClient
 from sync_amz_data.settings import AWS_LWA_CLIENT
 import pandas as pd
 import json
+from sync_amz_data.tasks.datainsert.wg import LADS
 
 
 class RateLimitError(Exception):
@@ -11,8 +12,8 @@ class RateLimitError(Exception):
         self.retry_after = retry_after
 
 
-def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None):
-    ADS = "http://192.168.1.23:8001/"
+def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None, AD = LADS):
+    ADS = AD
     resp = requests.session().request(
         method=method,
         url=urljoin(ADS, url_path),

sync_amz_data/tasks/datainsert/SB/mysql_datainsert_sbkeyword_v3.py

@@ -4,6 +4,7 @@ from sync_amz_data.public.amz_ad_client import SBClient
 from sync_amz_data.settings import AWS_LWA_CLIENT
 import pandas as pd
 import json
+from sync_amz_data.tasks.datainsert.wg import LADS
 
 
 class RateLimitError(Exception):
@@ -11,8 +12,8 @@ class RateLimitError(Exception):
         self.retry_after = retry_after
 
 
-def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None):
-    ADS = "http://192.168.1.23:8001/"
+def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None, AD = LADS):
+    ADS = AD
     resp = requests.session().request(
         method=method,
         url=urljoin(ADS, url_path),

sync_amz_data/tasks/datainsert/SB/mysql_datainsert_sbnegativekeyword.py

@@ -0,0 +1,89 @@
+import requests
+from urllib.parse import urljoin
+from sync_amz_data.public.amz_ad_client import SBClient
+from sync_amz_data.settings import AWS_LWA_CLIENT
+import pandas as pd
+import json
+from sync_amz_data.tasks.datainsert.wg import LADS
+
+
+class RateLimitError(Exception):
+    def __init__(self, retry_after: str = None):
+        self.retry_after = retry_after
+
+
+def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None, AD = LADS):
+    ADS = AD
+    resp = requests.session().request(
+        method=method,
+        url=urljoin(ADS, url_path),
+        headers=head,
+        params=params,
+        json=body,
+    )
+    if resp.status_code == 429:
+        raise RateLimitError(resp.headers.get("Retry-After"))
+    if resp.status_code >= 400:
+        raise Exception(resp.text)
+    return resp.json()
+
+
+class SbNegtiveKeyword:
+    def __init__(self, profile_id, campaignId: list = None):
+        self.profile_id = profile_id
+        self.portfolioId = campaignId
+        self.re_url_path = "api/ad_manage/profiles/"
+        self.upcreate_url_path = "api/ad_manage/sbnegativekeyword/updata/"
+        self.heads = {'X-Token': "da4ab6bc5cbf1dfa"}
+        self.refresh_token = self.get_refresh_token()
+        self.lwa_client_id = AWS_LWA_CLIENT['lwa_client_id']
+        self.lwa_client_secret = AWS_LWA_CLIENT['lwa_client_secret']
+        self.AWS_CREDENTIALS = {
+            'lwa_client_id': self.lwa_client_id,
+            'lwa_client_secret': self.lwa_client_secret,
+            'refresh_token': self.refresh_token,
+            'profile_id': self.profile_id
+        }
+
+    def get_refresh_token(self):
+        params = {'profile_id': self.profile_id}
+        heads = self.heads
+        url_path = self.re_url_path
+        tem = request(url_path=url_path, head=heads, params=params)
+        if tem.get('data') is not None:
+            _ = tem.get('data')
+            out = _[0].get('refresh_token')
+        else:
+            out = None
+        return out
+
+    def get_sbnegtivekeywords_data(self):
+        tem = SBClient(**self.AWS_CREDENTIALS)
+        _ = tem.get_negative_keyword()
+        df_sbnegtivekeywords = pd.json_normalize(_)
+        return df_sbnegtivekeywords
+
+    def dataconvert(self):
+        df = self.get_sbnegtivekeywords_data()
+        df['profile'] = self.profile_id
+        df.rename(columns={
+            'adGroupId': 'adGroup',
+            'campaignId': 'campaign',
+            'extendedData_creationDateTime': 'creationDateTime'
+        }, inplace=True)
+        json_data = json.loads(df.to_json(orient='records', force_ascii=False))
+        return json_data
+
+    def updata_create(self):
+        body = self.dataconvert()
+        heads = self.heads
+        url_path = self.upcreate_url_path
+        tem = request(url_path=url_path, head=heads, body=body, method="POST")
+        return tem
+
+
+if __name__ == '__main__':
+    a = SbNegtiveKeyword(profile_id="3006125408623189")
+    # out = a.get_sbnegtivekeywords_data()
+    out = a.updata_create()
+    print(out)

sync_amz_data/tasks/datainsert/SB/mysql_datainsert_sbtarget_v3.py

@@ -4,6 +4,7 @@ from urllib.parse import urljoin
 from sync_amz_data.settings import AWS_LWA_CLIENT
 import pandas as pd
 import json
+from sync_amz_data.tasks.datainsert.wg import LADS
 
 
 class RateLimitError(Exception):
@@ -11,8 +12,8 @@ class RateLimitError(Exception):
         self.retry_after = retry_after
 
 
-def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None):
-    ADS = "http://192.168.1.23:8001/"
+def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None, AD = LADS):
+    ADS = AD
     resp = requests.session().request(
         method=method,
         url=urljoin(ADS, url_path),

sync_amz_data/tasks/datainsert/SB/mysql_datainsert_sbtargetbid_recommendations_v3.py

@@ -4,6 +4,7 @@ from sync_amz_data.public.amz_ad_client import SBClient
 from sync_amz_data.settings import AWS_LWA_CLIENT
 import pandas as pd
 import json
+from sync_amz_data.tasks.datainsert.wg import LADS
 
 # pd.set_option('display.max_columns', None)
 # # 显示所有行
@@ -14,8 +15,8 @@ class RateLimitError(Exception):
         self.retry_after = retry_after
 
 
-def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None):
-    ADS = "http://192.168.1.23:8001/"
+def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None, AD = LADS):
+    ADS = AD
     resp = requests.session().request(
         method=method,
         url=urljoin(ADS, url_path),

sync_amz_data/tasks/datainsert/mysql_datainsert_sdcampaign.py → sync_amz_data/tasks/datainsert/SD/mysql_datainsert_sdcampaign.py

@@ -4,6 +4,7 @@ from sync_amz_data.public.amz_ad_client import SDClient
 from sync_amz_data.settings import AWS_LWA_CLIENT
 import pandas as pd
 import json
+from sync_amz_data.tasks.datainsert.wg import LADS
 
 
 class RateLimitError(Exception):
@@ -11,8 +12,8 @@ class RateLimitError(Exception):
         self.retry_after = retry_after
 
 
-def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None):
-    ADS = "http://192.168.1.23:8001/"
+def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None, AD = LADS):
+    ADS = AD
     resp = requests.session().request(
         method=method,
         url=urljoin(ADS, url_path),

sync_amz_data/tasks/datainsert/SP/mysql_datainsert_sp_budget_recommendation.py

@@ -4,6 +4,7 @@ from sync_amz_data.public.amz_ad_client import SPClient
 from sync_amz_data.settings import AWS_LWA_CLIENT
 import pandas as pd
 import json
+from sync_amz_data.tasks.datainsert.wg import LADS
 
 
 class RateLimitError(Exception):
@@ -11,8 +12,8 @@ class RateLimitError(Exception):
         self.retry_after = retry_after
 
 
-def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None):
-    ADS = "http://192.168.1.23:8001/"
+def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None, AD = LADS):
+    ADS = AD
     resp = requests.session().request(
         method=method,
         url=urljoin(ADS, url_path),

sync_amz_data/tasks/datainsert/SP/mysql_datainsert_sp_targetsbid_recommendations.py

@@ -4,6 +4,7 @@ from sync_amz_data.public.amz_ad_client import SPClient
 from sync_amz_data.settings import AWS_LWA_CLIENT
 import pandas as pd
 import json
+from sync_amz_data.tasks.datainsert.wg import LADS
 
 
 class RateLimitError(Exception):
@@ -11,8 +12,8 @@ class RateLimitError(Exception):
         self.retry_after = retry_after
 
 
-def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None):
-    ADS = "http://192.168.1.23:8001/"
+def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None, AD = LADS):
+    ADS = AD
     resp = requests.session().request(
         method=method,
         url=urljoin(ADS, url_path),

sync_amz_data/tasks/datainsert/SP/mysql_datainsert_sp_targetsbid_recommendations_v2.py

@@ -4,6 +4,7 @@ from sync_amz_data.public.amz_ad_client import SPClient
 from sync_amz_data.settings import AWS_LWA_CLIENT
 import pandas as pd
 import json
+from sync_amz_data.tasks.datainsert.wg import LADS
 
 
 class RateLimitError(Exception):
@@ -34,8 +35,8 @@ def convert_list(val):
     else:
         return val
 
-def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None):
-    ADS = "http://192.168.1.23:8001/"
+def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None, AD = LADS):
+    ADS = AD
     resp = requests.session().request(
         method=method,
         url=urljoin(ADS, url_path),

sync_amz_data/tasks/datainsert/SP/mysql_datainsert_spads.py

@@ -4,15 +4,15 @@ from sync_amz_data.public.amz_ad_client import SPClient
 from sync_amz_data.settings import AWS_LWA_CLIENT
 import pandas as pd
 import json
-
+from sync_amz_data.tasks.datainsert.wg import LADS
 
 class RateLimitError(Exception):
     def __init__(self, retry_after: str = None):
         self.retry_after = retry_after
 
 
-def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None):
-    ADS = "http://192.168.1.23:8001/"
+def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None, AD = LADS):
+    ADS = AD
     resp = requests.session().request(
         method=method,
         url=urljoin(ADS, url_path),

sync_amz_data/tasks/datainsert/SP/mysql_datainsert_spcampaign.py

@@ -4,15 +4,15 @@ from sync_amz_data.public.amz_ad_client import SPClient
 from sync_amz_data.settings import AWS_LWA_CLIENT
 import pandas as pd
 import json
-
+from sync_amz_data.tasks.datainsert.wg import LADS
 
 class RateLimitError(Exception):
     def __init__(self, retry_after: str = None):
         self.retry_after = retry_after
 
 
-def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None):
-    ADS = "http://192.168.1.23:8001/"
+def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None, AD = LADS):
+    ADS = AD
     resp = requests.session().request(
         method=method,
         url=urljoin(ADS, url_path),

sync_amz_data/tasks/datainsert/SP/mysql_datainsert_spgroup.py

@@ -4,15 +4,15 @@ from sync_amz_data.public.amz_ad_client import SPClient
 from sync_amz_data.settings import AWS_LWA_CLIENT
 import pandas as pd
 import json
-
+from sync_amz_data.tasks.datainsert.wg import LADS
 
 class RateLimitError(Exception):
     def __init__(self, retry_after: str = None):
         self.retry_after = retry_after
 
 
-def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None):
-    ADS = "http://192.168.1.23:8001/"
+def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None, AD = LADS):
+    ADS = AD
     resp = requests.session().request(
         method=method,
         url=urljoin(ADS, url_path),

sync_amz_data/tasks/datainsert/SP/mysql_datainsert_spkeyword.py

@@ -4,15 +4,15 @@ from sync_amz_data.public.amz_ad_client import SPClient
 from sync_amz_data.settings import AWS_LWA_CLIENT
 import pandas as pd
 import json
-
+from sync_amz_data.tasks.datainsert.wg import LADS
 
 class RateLimitError(Exception):
     def __init__(self, retry_after: str = None):
         self.retry_after = retry_after
 
 
-def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None):
-    ADS = "http://192.168.1.23:8001/"
+def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None, AD = LADS):
+    ADS = AD
     resp = requests.session().request(
         method=method,
         url=urljoin(ADS, url_path),

sync_amz_data/tasks/datainsert/SP/mysql_datainsert_spnegativekeyword.py

@@ -4,15 +4,15 @@ from sync_amz_data.public.amz_ad_client import SPClient
 from sync_amz_data.settings import AWS_LWA_CLIENT
 import pandas as pd
 import json
-
+from sync_amz_data.tasks.datainsert.wg import LADS
 
 class RateLimitError(Exception):
     def __init__(self, retry_after: str = None):
         self.retry_after = retry_after
 
 
-def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None):
-    ADS = "http://192.168.1.23:8001/"
+def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None, AD = LADS):
+    ADS = AD
     resp = requests.session().request(
         method=method,
         url=urljoin(ADS, url_path),

sync_amz_data/tasks/datainsert/SP/mysql_datainsert_spnegativetarget.py

@@ -6,15 +6,15 @@ from sync_amz_data.public.amz_ad_client import SPClient
 from sync_amz_data.settings import AWS_LWA_CLIENT
 import pandas as pd
 import json
-
+from sync_amz_data.tasks.datainsert.wg import LADS
 
 class RateLimitError(Exception):
     def __init__(self, retry_after: str = None):
         self.retry_after = retry_after
 
 
-def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None):
-    ADS = "http://192.168.1.23:8001/"
+def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None, AD = LADS):
+    ADS = AD
     resp = requests.session().request(
         method=method,
         url=urljoin(ADS, url_path),

sync_amz_data/tasks/datainsert/SP/mysql_datainsert_sptarget.py

@@ -6,15 +6,15 @@ from sync_amz_data.public.amz_ad_client import SPClient
 from sync_amz_data.settings import AWS_LWA_CLIENT
 import pandas as pd
 import json
-
+from sync_amz_data.tasks.datainsert.wg import LADS
 
 class RateLimitError(Exception):
     def __init__(self, retry_after: str = None):
         self.retry_after = retry_after
 
 
-def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None):
-    ADS = "http://192.168.1.23:8001/"
+def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None, AD = LADS):
+    ADS = AD
     resp = requests.session().request(
         method=method,
         url=urljoin(ADS, url_path),

sync_amz_data/tasks/datainsert/alldata_insert.py

@@ -17,6 +17,7 @@ from SB.mysql_datainsert_sbads import SbAds
 from SB.mysql_datainsert_sb_keywordsbid_recommendations_v3 import SbkeywordsBidRecommendations
 from SB.mysql_datainsert_sbtarget_v3 import SbTargets
 from SB.mysql_datainsert_sbtargetbid_recommendations_v3 import SbtargetsBidRecommendations
+from SB.mysql_datainsert_sbnegativekeyword import SbNegtiveKeyword
 
 import time
 
@@ -139,4 +140,8 @@ sbtbro = sbtbr.updata_create()
 print("SbtargetsBidRecommendations", sbtbro)
 protime(start_time)
 
-
+start_time = time.time()
+Sbnk = SbNegtiveKeyword(profile_id="3006125408623189")
+Sbnko = Sbnk.updata_create()
+print("SbNegtiveKeyword", Sbnko)
+protime(start_time)

sync_amz_data/tasks/datainsert/mysql_datainsert_portfolios.py

@@ -5,14 +5,15 @@ from sync_amz_data.settings import AWS_LWA_CLIENT
 import pandas as pd
 import json
 import time
+from sync_amz_data.tasks.datainsert.wg import LADS
 
 class RateLimitError(Exception):
     def __init__(self, retry_after: str = None):
         self.retry_after = retry_after
 
 
-def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None):
-    ADS = "http://192.168.1.23:8001/"
+def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None, AD = LADS):
+    ADS = AD
     resp = requests.session().request(
         method=method,
         url=urljoin(ADS, url_path),

sync_amz_data/tasks/datainsert/wg.py

@@ -0,0 +1 @@
+LADS = "http://192.168.1.39:8001/"