聚美优品APP8.725接口分析获得商家列表数据
0x01、 目标需求:
.a) a) 分析聚美优品APP的接口参数以及相关的签名获取的方式
.b) b) 需要获取全品类分类接口,以及分类下商品列表接口
0x02、分析背景:
.a) 聚美优品APP版本V8.725(目前最新版)
.b) 软件无壳
.c) 软件通讯过程中无数据加密,反爬策略仅针对IP访问频率
0x03、分析流程:
.a) 通过数据包抓取或敏感函数hook方式获得接口功能
.b) 通过函数内部参数的组装继续分析参数的来源以及加密的流程
0x04、接口分析:
.a) 分类接口抓包分析：
0x001.分类api接口 
0x002. 返回数据解析(以下链接抓到的获取分类数据的接口信息.包含所有分类数据，图片，分类ID，一级、二级、三级、分类)
0x003. JSON数据解析Python打印后，如下图:
0x004. 分类的商品列表接口数据抓取，如下图:
Python代码如下,直接可以获取数据
import requests,json,re,time,urllib.parse
requests.packages.urllib3.disable_warnings()
headers = {
'User-Agent': 'JuMei/8.725 (M2003J15SC; Android; Android OS ; 10; zh) ApacheHttpClient/4.0',
'X-Jumei-Authorization': '',
'Host': 'api.jumei.com',
'Accept-Encoding': 'GZIP',
'X-Online-Host': 'api.jumei.com',
'Sm-Device-Id': '20200716113240e5fcda73e076fbc4d42013e9105cdfee00fc944ffba6f31b',
'X-Tingyun-Id': 'Dh-NvlyTByI;c=2;r=1500178814;',
'x_jm_owl_req_id': 'originalRequest.id',
'Connection': 'Keep-Alive',
'Cookie': 'default_site_25=bj; JSESSIONID=BDAA47A22F344C9DD9639BD7CFB27421; language=zh; source=qq-appstore; resolution=1080*2110; platform=android; mac=6ae449329df4c1ca; network=wifi; httpDnsVersion=26; httpDnsCase=1; model=M2003J15SC; unique_device_id=Xw/KTwrssXEDAAS3xFry5L5Z; PHPSESSID=01c2027bd6274afc788b2f78fd8f1d94; install_source=qq-appstore; brand=Redmi; user_ip=192.168.1.3; ab=3:d|7:e|123:b|131:d|150:a|160:c|164:v8|166:b|177:video2t10|179:video3t2|186:video4t19|191:c|281:v8|666:b|703:wvgre_a|909:a|1001:b|1200:a10|1300:normal|1302:a10|1655:v8|1806:normal|1807:j|1808:a19|1809:a19|1810:normal|6680:a|8998:b|9081:c|9108:videofeedb|9902:f|73163:d|73164:f; product=jumei; device_id=6ae449329df4c1ca; uc_api_session_id=5f0fca58466394937; user_tag_id=131; client_v=8.725; postcode=110000; platform_v=10; appfirstinstall=1; site=bj; device_uid=59df0cad-856c-4e49-a3dd-39f1888cb3c1; referer_site=app_android_com.jm.android.jumei_qq-appstore_v8.725_site; appid=com.jm.android.jumei; appsecret=114ab1fa; imei=6ae449329df4c1ca; android_id=6ae449329df4c1ca; is_first_open=1'
}
# 获取分类信息api
categoryUrl = "https://api.jumei.com/Search/Category?ab=3%3Ad%7C7%3Ae%7C123%3Ab%7C131%3Ad%7C150%3Aa%7C160%3Ac%7C164%3Av8%7C166%3Ab%7C177%3Avideo2t10%7C179%3Avideo3t2%7C186%3Avideo4t19%7C191%3Ac%7C281%3Av8%7C666%3Ab%7C703%3Awvgre_a%7C909%3Aa%7C1001%3Ab%7C1200%3Aa10%7C1300%3Anormal%7C1302%3Aa10%7C1655%3Av8%7C1806%3Anormal%7C1807%3Aj%7C1808%3Aa19%7C1809%3Aa19%7C1810%3Anormal%7C6680%3Aa%7C8998%3Ab%7C9081%3Ac%7C9108%3Avideofeedb%7C9902%3Af%7C73163%3Ad%7C73164%3Af&android_id=6ae449329df4c1ca&antifraud_sign=d06f112b5c34a4ce44d31ea3d9d2f1b3&antifraud_tid=3615756017&antifraud_ts="+str(int(time.time()))+"&appfirstinstall=1&client_v=8.725&global_init_log_device_model=m2003j15sc&global_init_log_device_vendor=xiaomi&global_init_log_os_version=10&global_init_log_push_enable=on&global_init_log_push_id=bd02330118a9043c61cc88a287e630c9&is_first_launch=0&is_first_open=1&platform=android&site=bj&source=qq-appstore&unique_device_id=Xw%2FKTwrssXEDAAS3xFry5L5Z&user_tag_id=131"
# 获取商品列表api
shopListUrl = "https://api.jumei.com/Search/Data?ab=3%3Ad%7C7%3Ae%7C123%3Ab%7C131%3Ad%7C150%3Aa%7C160%3Ac%7C164%3Av8%7C166%3Ab%7C177%3Avideo2t10%7C179%3Avideo3t2%7C186%3Avideo4t19%7C191%3Ac%7C281%3Av8%7C666%3Ab%7C703%3Awvgre_a%7C909%3Aa%7C1001%3Ab%7C1200%3Aa10%7C1300%3Anormal%7C1302%3Aa10%7C1655%3Av8%7C1806%3Anormal%7C1807%3Aj%7C1808%3Aa19%7C1809%3Aa19%7C1810%3Anormal%7C6680%3Aa%7C8998%3Ab%7C9081%3Ac%7C9108%3Avideofeedb%7C9902%3Af%7C73163%3Ad%7C73164%3Af&android_id=6ae449329df4c1ca&antifraud_sign=d22dfff9041604f361eb9d7db117f7a7&antifraud_tid=3615756017&antifraud_ts="+str(int(time.time()))+"&appfirstinstall=1&auto_correct=1&category_id={}&client_v=8.725&global_init_log_device_model=m2003j15sc&global_init_log_device_vendor=xiaomi&global_init_log_os_version=10&global_init_log_push_enable=on&global_init_log_push_id=bd02330118a9043c61cc88a287e630c9&is_first_launch=0&is_first_open=1&is_first_search=1&is_together_filter_brand=true&item_per_page=20&page={}&platform=android&search={}&search_source=category&search_source_ex=main_search&search_type=mSearch&site=bj&source=qq-appstore&unique_device_id=Xw%2FKTwrssXEDAAS3xFry5L5Z&user_tag_id=131"
def get_category():
res = requests.get(url=categoryUrl,headers=headers,verify=False)
return res
def get_item_list(categoryId,page,keyWord):
url = shopListUrl.format(categoryId,page,keyWord)
res = requests.get(url=url,headers=headers,verify=False)
j = json.loads(res.text)['data']
print("\t|\t\t\t|\t\t\t|")
try:
print("\t|\t\t\t|\t\t\t|=====================>当前:页码[%s],总页数[%s],商品数量[%s],每页条数[%s]" % (j['page'],j['page_count'],j['item_count'],j['item_per_page']))
except:
print(res.text)
print(url)
for i in j['item_list_component']:
for t in i['title']:
if t['type'] == 'main':
title = t['desc']
print("\t|\t\t\t|\t\t\t|============================>当前:商品ID[%s],商品标题[%s],商品类型[%s]" % (i['info']['item_id'],title,i['info']['type']))
page = int(page)
if page < int(j['page_count']):
page += 1
get_item_list(categoryId,page,keyWord)
if __name__ == '__main__':
res = get_category()
data = json.loads(res.text)
category = data['data']['category']
for i in category:
if ('推荐' in i['name']) or ('热门' in i['name']) or ('热门' in i['name']):
continue
print("\t%s" % i['name'])
for j in i['second_level']:
if ('推荐' in j['name']) or ('热门' in j['name']) or ('大牌' in j['name']) or ('品牌' in j['name']):
continue
print("\t|")
print("\t|-------->%s" % j['name'])
for k in j['three_level']:
print("\t|\t\t\t|")
print("\t|\t\t\t|-------->%s[分类ID:%s]" % (k['name'],re.findall("\d+",k['link'])[0]))
get_item_list(re.findall("\d+",k['link'])[0],"1",urllib.parse.quote(k['name']))
在获取商品列表页时有三个参数其中有一个是签名,但是服务器并没有做校验.签名生成代码如下Java版,主要是对uri进行加密后再吧参数追加进去,暂时不需要.import org.apache.commons.lang3.StringUtils;
import java.net.URLDecoder;
import java.security.MessageDigest;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
public class JumeiParamTest {
public static void main(String[] args) {
long currentUnixTime = System.currentTimeMillis()/1000;
String url = "https://api.jumei.com/Search/Data?";
String uri = "source=qq-appstore&global_init_log_device_model=m2003j15sc&platform=android&is_first_launch=0&search=女香&category_id=35&global_init_log_os_version=10&search_source_ex=main_search&unique_device_id=Xw/KTwrssXEDAAS3xFry5L5Z&auto_correct=1&ab=3:d|7:e|123:b|131:d|150:a|160:c|164:v8|166:b|177:video2t10|179:video3t2|186:video4t19|191:c|281:v8|666:b|703:wvgre_a|909:a|1001:b|1200:a10|1300:normal|1302:a10|1655:v8|1806:normal|1807:j|1808:a19|1809:a19|1810:normal|6680:a|8998:b|9081:c|9108:videofeedb|9902:f|73163:d|73164:f&user_tag_id=131&client_v=8.725&is_together_filter_brand=true&global_init_log_push_id=bd02330118a9043c61cc88a287e630c9&global_init_log_push_enable=on&search_source=category&search_type=mSearch&item_per_page=20&appfirstinstall=1&site=bj&is_first_search=1&page=1&android_id=6ae449329df4c1ca&global_init_log_device_vendor=xiaomi&is_first_open=1";
Map<String,String> map = strToMap(uri);
map.remove("uid");
map.remove("antifraud_sign");
map.remove("antifraud_ts");
map.remove("antifraud_tid");
Object[] keys = map.keySet().toArray();
Arrays.sort(keys);
StringBuilder sb = new StringBuilder();
for (Object key : keys) {
if (!StringUtils.isEmpty(map.get(key))) {
sb.append(map.get(key));
}
}
sb.append(currentUnixTime);
sb.append("6b271087cea77e53b635fd24cb99525bv:v3");
String s = sb.toString();
System.out.println(s);
try {
byte[] btInput = s.getBytes("UTF-8");
MessageDigest md5 = MessageDigest.getInstance("MD5");
md5.update(btInput);
byte[] bs = md5.digest();
sb = new StringBuilder();
for (byte b2 : bs) {
sb.append(Integer.toHexString((b2 & -1) | -256).substring(6));
}
System.out.println(sb.toString());
} catch (Exception e2) {
e2.printStackTrace();
}
}
public static Map<String,String> strToMap(String str){
String[] strs = str.split("&");
Map<String,String> m = new HashMap<>();
StringBuffer sb = new StringBuffer();
for (String s :strs){
m.put(s.split("=")[0],s.split("=")[1]);
}
return m;
}
}
待解决问题:爬取商品列表页不能超过100页,超过后不返回数据
友情提示:本文只为技术分享交流,请勿非法用途.产生一切法律问题与本人无关
在浏览的同时希望给予作者打赏,来支持作者的服务器维护费用.一分也是爱~
