output:
result = self.fn(*self.args, **self.kwargs)
File "app-1.py", line 103, in run
results.append(self.find_keyword(keyword, country))
File "app-1.py", line 75, in find_keyword
for i, search_item in enumerate(search_items, start=1):
TypeError: 'NoneType' object is not iterable
my code:
import undetected_chromedriver as uc
uc.install()
import sys, os
import traceback
import requests
import urllib.parse as p
import pandas as pd
import datetime
from bs4 import BeautifulSoup
import random
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.chrome.options import Options
from time import sleep
import time
import re
import threading
import concurrent.futures
import csv
from config import *
import math
import code
date = datetime.datetime.now()
date_format = date.strftime("%d-%m-%Y, %H:%M:%S")
date_only = date.strftime("%d-%m-%Y-%H%M%S")
import logging
import sys
logger = logging.getLogger()
logger.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s | %(levelname)s | %(message)s',
'%m-%d-%Y %H:%M:%S')
file_handler = logging.FileHandler(f'logs/{date_only}.log', encoding="utf8")
file_handler.setLevel(logging.DEBUG)
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
class Process:
def __init__(self, keywords, url):
self.keywords = keywords
#self.no_of_pages = no_of_pages
self.count = 1
self.url = url
self.driver = None
def find_keyword(self, keyword,country):
for page in range(1, 11):
print("[*] Going for page:", page)
# calculating start
start = (page - 1) * 10 + 1
# make API request
url =
f"https://www.googleapis.com/customsearch/v1?key={API_KEY}&cx={SEARCH_ENGINE_ID}&q={keyword}&gl={country}&start={start}";
data = requests.get(url).json()
search_items = data.get("items")
# a boolean that indicates whether `target_domain` is found
found = False
for i, search_item in enumerate(search_items, start=1):
# get the page title
title = search_item.get("title")
# page snippet
snippet = search_item.get("snippet")
# alternatively, you can get the HTML snippet (bolded
keywords)
html_snippet = search_item.get("htmlSnippet")
# extract the page url
link = search_item.get("link")
# extract the domain name from the URL
domain_name = p.urlparse(link).netloc
if domain_name.endswith(self.url):
# get the page rank
rank = i + start - 1
print(f"[+] {self.url} is found on rank #{rank} for
keyword: '{keyword}'")
print("[+] Title:", title)
print("[+] Snippet:", snippet)
print("[+] URL:", link)
logging.info(f"link -- {link}")
return {'keyword': keyword, 'rank' : rank}
# target domain is found, exit out of the program
else:
return {'keyword': keyword, 'rank' : 0}
def run(self):
results = []
for keyword, country in self.keywords:
results.append(self.find_keyword(keyword, country))
return results
if __name__=="__main__":
result = []
df = pd.read_csv('Input/Keywords - Sheet2.csv')
keywords = list(df['keyword'].values)
countries = list(df['country'].values)
kc = list(zip(keywords,countries))
url = "vpnbrains.com"
row = len(keywords)
arr = []
batch = BATCH
batchsize = math.ceil(row/batch)
for i in range(batch):
arr.append(kc[i*batchsize:(i+1)*batchsize])
with concurrent.futures.ThreadPoolExecutor() as executor:
futures = []
for af in arr :
if (i+1)*batchsize > row:
obj = Process(kc,url)
futures.append(
executor.submit(
obj.run
)
)
else:
obj = Process(af,url)
futures.append(
executor.submit(
obj.run
)
)
for j in futures:
res = j.result()
result.extend(res)
out_df = pd.DataFrame(result)
out_df.to_csv(f'Export/{date_only}.csv', index=False)
code.interact(local=locals())
--
You received this message because you are subscribed to the Google Groups
"Google App Engine" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
To view this discussion on the web visit
https://groups.google.com/d/msgid/google-appengine/1d4990cc-1232-4d8c-9328-29dbe458193bn%40googlegroups.com.
