Merge branch 'wengao' of ASJ_ADS/sync_amz_data into master

sync_amz_data/DataTransform/Data_ETL.py

@@ -571,6 +571,10 @@ class SB_ETL(SBClient, Common_ETLMethod):
         # df_targets = self.TZ_Deal(df_targets, ["extendedData.creationDateTime", "extendedData.lastUpdateDateTime"])
         # df_targets = self.expression_split(df_targets, "resolvedExpressions")
         df_targets = self.id_type_trans(df_targets)
+        df_targets['expressions_type'] = df_targets['expressions'].map(
+            lambda x: self.get_keyOvalue(x, 'type'))
+        df_targets['expressions_value'] = df_targets['expressions'].map(
+            lambda x: self.get_keyOvalue(x, 'value'))
         df_targets['resolvedExpressions_type'] = df_targets['resolvedExpressions'].map(lambda x:self.get_keyOvalue(x,'type'))
         df_targets['resolvedExpressions_value'] = df_targets['resolvedExpressions'].map(lambda x:self.get_keyOvalue(x,'value'))
         return self.columnsName_modify(df_targets)

sync_amz_data/tasks/datainsert/SB/mysql_datainsert_sbtargetbid_recommendations_v3.py

@@ -48,6 +48,8 @@ def convert_row(row):
     return expressions
 
 def convert_list(val):
+    if val is None:
+        return []
     if '[' in val:
         return val.split('[')[1].split(']')[0].split(',')
     else:
@@ -116,12 +118,12 @@ class SbtargetsBidRecommendations:
             res = tem.iter_target_bidrecommendation(**i)
             out_data.extend(list(res))
         temtest = pd.json_normalize(out_data)
-        temtest['targets'] = temtest['targets'].astype(str).str.lower()
-        df_old['targets'] = df_old['targets'].astype(str).str.lower()
+        temtest['targets'] = temtest['targets'].fillna('').astype(str).str.lower()
+        df_old['targets'] = df_old['targets'].fillna('').astype(str).str.lower()
         _ = pd.merge(left=df_old, right=temtest, on=['targets', 'campaignId'], how='left')
         outall = _[['targetId', 'recommendedBid.rangeStart', 'recommendedBid.rangeEnd',
                     'recommendedBid.recommended']].copy()
-        outall = outall.dropna(subset=['recommendedBid.rangeEnd'])
+        # outall = outall.dropna(subset=['recommendedBid.rangeEnd'])
         outall.rename(columns={'recommendedBid.recommended': 'suggestedBid',
                                'recommendedBid.rangeStart': 'suggestedBid_lower',
                                'recommendedBid.rangeEnd': 'suggestedBid_upper',