Java爬取百度云观测对网站的检测数据，获取子域名及域名的安全信息

JSON 2020-03-31 20:12:21 7814

百度云观测有对域名的检测还比较全面。下面我们来抓取下内容看看。

Java抓取代码

直接上代码，HTTP工具类暂时不能提供给您，您随便自己请求就可以了，只要能通，能返回数据。

//获取子域名 & 子域名评分
public List<DomainCheckInfoDto.SSLDomain> SSLDomains(String surl){
    String domain = StringUtils.getHost(surl);

    surl = String.format("http://ce.baidu.com/index/getRelatedSites?site_address=%s", StringUtils.strToUrlcode(surl))      ;

    List<DomainCheckInfoDto.SSLDomain> domainList = new ArrayList<>();
    String res = Http.create(surl).
            head("Accept", "application/json, text/javascript, */*; q=0.01")
            .head("Accept-Encoding", "gzip, deflate")
            .head("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8")
            .head("Cache-Control", "no-cache")
            .head("Connection", "keep-alive")
            .head("Cookie", "您的百度登录Cookie")
            .head("Host", "ce.baidu.com")
            .head("Pragma", "no-cache")
            .head("Referer", String.format("http://ce.baidu.com/index/guance?start_url=%s", domain))
            .head("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36")
            .head("X-Requested-With", "XMLHttpRequest")
            .timeout(3)
            .readTimeout(3)
            .get().send().getResponse().getResult();
    if(StringUtils.isNotBlank(res) ){

        JSONObject json = JSONObject.fromObject(res);
        int code = json.optInt("code", 1);
        if(0 == code){//成功
            JSONArray data = json.optJSONArray("data");

            //最多弄10个域名，有的时候子域名太多了
            for (int i = 0; i < data.size() && i< 10; i++) {
                Object datum = data.get(i);
                JSONObject target = null;
                if(datum instanceof  JSONObject){
                    target = (JSONObject)datum;
                }else{
                    target = JSONObject.fromObject(datum);
                }
                DomainCheckInfoDto.SSLDomain sslDomain = new DomainCheckInfoDto.SSLDomain(target);

                domainList.add(sslDomain);
            }

        }
    }
    return domainList;
}

然后里面有一个实体类：

@Data
@AllArgsConstructor
@NoArgsConstructor
public static class SSLDomain{
    //分值
    private int  score;
    //域名
    private String  domain;

    public SSLDomain(JSONObject obj){
        this.score = obj.optInt("score", 60);
        this.domain = obj.optString("domain", "");
    }
}

数据返回：

{
    "attackSSL":{
        "name":"攻击风险",
        "score":75,
        "scoreDelta":0
    },
    "cateId":"medial",
    "cateName":"影视音乐",
    "cateScore":60,
    "domainLastScore":54,
    "domainSize":10,
    "domsins":[
        {
            "domain":"air.sojson.com",
            "score":84
        },
        {
            "domain":"api.sojson.com",
            "score":54
        },
        {
            "domain":"css.sojson.com",
            "score":80
        },
        {
            "domain":"fm.sojson.com",
            "score":80
        },
        {
            "domain":"icp.sojson.com",
            "score":94
        },
        {
            "domain":"js.sojson.com",
            "score":65
        },
        {
            "domain":"m.sojson.com",
            "score":80
        },
        {
            "domain":"open.sojson.com",
            "score":90
        },
        {
            "domain":"sojson.com",
            "score":90
        },
        {
            "domain":"t.weather.sojson.com",
            "score":80
        }
    ],
    "envSSL":{
        "name":"网站服务",
        "score":40,
        "scoreDelta":0
    },
    "historySSL":{
        "name":"网站历史安全",
        "score":100,
        "scoreDelta":0
    },
    "id":"17bb1d64a239c585",
    "rank":0,
    "realtimeSSL":{
        "name":"网站恶意内容",
        "score":75,
        "scoreDelta":0
    },
    "synopsis":73,
    "updatedTime":"2020-03-31 17:51:27"
}

这个是查询SOJSON.COM的结果