python + selenium + phantomjs ne peut pas gérer https url qui `fortement ajouter la certification '

Question

J'essaie d'utiliser python + selenium + phantomjs gérer la demande https, mais ne pas recevoir de réponse du serveur qui me montre adding certification is strongly advised, voici mon Fonction get_request.

def get_random_x_forwarded_for(): 
    # 得到随机x-forwarded-for值 
    numbers = [] 
    while not numbers or numbers[0] in (10, 172, 192): 
     numbers = random.sample(range(1, 255), 4) 
    return '.'.join(str(_) for _ in numbers) 

def get_random_ua(): 
    # 得到随机user-agent值 
    import os 
    if os.path.exists("dicts/user-agent.txt"): 
     f = open(ModulePath + "dicts/user-agents.txt", "r+") 
     all_user_agents = f.readlines() 
     f.close() 
    else: 
     all_user_agents = [ 
      "Mozilla/4.0 (Mozilla/4.0; MSIE 7.0; Windows NT 5.1; FDM; SV1)", 
      "Mozilla/4.0 (Mozilla/4.0; MSIE 7.0; Windows NT 5.1; FDM; SV1; .NET CLR 3.0.04506.30)", 
      "Mozilla/4.0 (Windows; MSIE 7.0; Windows NT 5.1; SV1; .NET CLR 2.0.50727)", 
      "Mozilla/4.0 (Windows; U; Windows NT 5.0; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.195.33 Safari/532.0", 
      "Mozilla/4.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/1.0.154.59 Safari/525.19", 
      "Mozilla/4.0 (compatible; MSIE 6.0; Linux i686 ; en) Opera 9.70", 
     ] 
    random_ua_index = random.randint(0, len(all_user_agents) - 1) 
    ua = re.sub(r"(\s)$", "", all_user_agents[random_ua_index]) 
    return ua 

def get_request(url, by="MechanicalSoup", proxyUrl="", cookie=""): 
    code = None 
    title = None 
    content = None 
    if by == "seleniumPhantomJS": 
     hasFormAction=False 
     formActionValue="" 
     from selenium import webdriver 
     from selenium.common.exceptions import TimeoutException 
     import time 
     if proxyUrl == "" or proxyUrl == 0: 
      service_args_value = ['--ignore-ssl-errors=true', '--ssl-protocol=any'] 
     if proxyUrl != "" and proxyUrl != 0: 
      proxyType = proxyUrl.split(":")[0] 
      proxyValueWithType = proxyUrl.split("/")[-1] 
      service_args_value = ['--ignore-ssl-errors=true', '--ssl-protocol=any', 
            '--proxy=%s' % proxyValueWithType, '--proxy-type=%s' % proxyType] 
     # final_url=driver.current_url 
     # print("正在访问的url是这个:\n"+final_url) 
     # driver.quit() 

     try: 
      from selenium.webdriver.common.desired_capabilities import DesiredCapabilities 
      if cookie != "": 
       dcap = dict(DesiredCapabilities.PHANTOMJS) 
       dcap["phantomjs.page.settings.cookie"] = cookie 
       dcap["phantomjs.page.settings.userAgent"] = get_random_ua() 
       driver = webdriver.PhantomJS(service_args=service_args_value, desired_capabilities=dcap) 
      else: 
       driver = webdriver.PhantomJS(service_args=service_args_value) 

      print(111111111) 
      driver.implicitly_wait(5) 
      driver.set_page_load_timeout(5) 

      driver.get(url) 
      print(222222222222222) 
      # http://www.cnblogs.com/fnng/p/3269450.html 
      originalCookie = driver.get_cookies() 

      #print("current cookie is:\n" + str(originalCookie)) 

      import random 
      code = 200 
      title = driver.title 
      content = driver.page_source 
      a=re.search(r'''(<.*type=('|")?submit('|")?.*>)''',content,re.I) 
      if a: 
       hasFormAction=True 
       print(a.group(1)) 
       input(67666) 
      else: 
       pass 

      print("len content is :\n" + str(len(content))) 
      print("title is :\n" + title) 

      if re.search(r"(页面不存在)|(未找到页面)|(page not found)|(404)",title+content,re.I): 
       return get_request(url,by="MechanicalSoup") 
      # time.sleep(5) # Let the user actually see something! 
      # driver.quit() 

     except TimeoutException as e: 
      # Handle your exception here 
      print(e) 
     finally: 
      driver.quit() 

     return { 
      'code': code, 
      'title': title, 
      'content': content, 
      #True or False 
      'hasFormAction':hasFormAction, 
      #eg,https://www.baidu.com^a=1&b=2 
      #eg,https://www.baidu.com/?a=1&b=2 
      'formActionValue':formActionValue} 

    else: 
     import mechanicalsoup 

     try: 
      browser = mechanicalsoup.Browser(soup_config={"features": "lxml"}) 
      ua = get_random_ua() 
      browser.session.headers.update({'User-Agent': '%s' % ua}) 
      # headers=browser.session.headers 
      # if 'Cookie' in headers: 
      # originalCookie=headers['Cookie'] 
      if cookie == "": 
       pass 
      else: 
       browser.session.headers.update({'Cookie': '%s' % cookie}) 
      # print(originalCookie) 
      x_forwarded_for = get_random_x_forwarded_for() 
      browser.session.headers.update(
       {'X-Forwarded-For': '%s' % x_forwarded_for}) 


      result = browser.get(url, 
        timeout=10,verify=False) 
      # print(dir(result)) 
      code = result.status_code 
      content = result.content 
      import chardet 
      bytesEncoding = chardet.detect(content)['encoding'] 
      # print(bytesEncoding) 
      content = content.decode(bytesEncoding) 
      title = BeautifulSoup(content, "lxml").title 
      if title is not None: 
       title_value = title.string 
      else: 
       title_value = None 
     except: 

      code = 0 
      title_value = "you may be blocked or the code doesn't handle ssl certificate well" 
      content = 'can not get html content this time,may be blocked by the server to request' 

     return_value = { 
      'code': code, 
      'title': title_value, 
      'content': content} 
     # print("访问当前url为:\n\t"+url+"\ntitle如下:") 
     # print("\t"+str(return_value['title'])) 
     return return_value 

b=get_request("https://www.zoomeye.org/search/advanced",by="seleniumPhantomJS") 
#b=get_request("https://www.zoomeye.org/search/advanced",by="MechanicalSoup") 
print(b) 

quand j'utilise b=get_request("https://www.zoomeye.org/search/advanced",by="seleniumPhantomJS"), elle imprime uniquement 11111..1 et coincé là-bas sans impression 2222,2, alors je tente d'utiliser b=get_request("https://www.zoomeye.org/search/advanced",by="MechanicalSoup"), cette fois, je reçois ci-dessous les informations de débogage d'erreur:

/usr/local/lib/python3.6/site-packages/requests/packages/urllib3/connectionpool.py:852: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings InsecureRequestWarning)

Je veux utiliser python3 + selenium + phantomjs pour gérer cette erreur adding certificate verification is strongly advised, quelqu'un peut-il m'aider?

Plus tard, j'ai essayé d'utiliser PhantomJS directement sans sélénium:

phantomjs --ignore-ssl-errors=true --ssl-protocol=any --web-security=false 1.js 1.js a ci-dessous le contenu:

var webPage = require('webpage'); 
var page = webPage.create(); 
console.log('test666'); 
page.open('https://www.zoomeye.org/search/advanced', function (status) { 
    var content = page.content; 
    console.log('Content: ' + content); 
    phantom.exit(); 
}); 

Cependant, cela ne fonctionne pas encore, peut-être est-ce bug de PhantomJS :(

Answer 1

Aller à travers l'URL ci-dessous Il vous fournira les capacités que vous devez utiliser https avec PhantomJS: -.

http://phantomjs.org/api/command-line.html

capacités principales vous sont nécessaires comme ci-dessous: -

• --web-security = [true | false] permet la sécurité Web et interdit XHR inter-domaines (par défaut est vrai). Également accepté: [oui | non].
• --ssl-protocol = [sslv3 | sslv2 | tlsv1 | tlsv1.1 | tlsv1.2 | any '] définit le protocole SSL pour les connexions sécurisées (SSLv3 par défaut). Toutes les valeurs peuvent ne pas être prises en charge, selon la bibliothèque OpenSSL du système. --ignore-ssl-errors = [true | false] ignore les erreurs SSL, telles que les erreurs de certificat expirées ou auto-signées (la valeur par défaut est false). Également accepté: [oui | non].

J'ai un code java qui fonctionne bien pour moi: -

private static Capabilities getPhantomCapabilities(String OS) { 
    DesiredCapabilities capabilities = null; 
    ArrayList<String> cliArgsCap = new ArrayList<String>(); 
    capabilities = DesiredCapabilities.phantomjs(); 
    cliArgsCap.add("--web-security=false"); 
    cliArgsCap.add("--ssl-protocol=any"); 
    cliArgsCap.add("--ignore-ssl-errors=true"); 
    capabilities.setCapability("takesScreenshot", true); 
    capabilities.setJavascriptEnabled(true); 
    capabilities.setCapability(
     PhantomJSDriverService.PHANTOMJS_CLI_ARGS, cliArgsCap); 
    capabilities.setCapability(
     PhantomJSDriverService.PHANTOMJS_GHOSTDRIVER_CLI_ARGS, 
      new String[] { "--logLevel=2" }); 
    return capabilities; 
} 

Convertir votre code en python. Il devrait fonctionner

espère que cela vous aidera :)

Answer 2

兄弟 你 的 https 代理 能用 吗, 我 用 http 的 代理 可以 使用 但是 用 https 的 代理 都 用不了, demande python 用 使用 https 代理 是没 问题 的, 使用 代码 如下

service_args = [ 
    '--proxy='+ip[1], 
    '--proxy-type=https', 
] 
driver = webdriver.PhantomJS(executable_path= os.getcwd()+'/driver/phantomjs.exe',service_args=service_args) #加载网址