57 lines
1.5 KiB
Python
57 lines
1.5 KiB
Python
|
'''Run electron BRE loading'''
|
||
|
|
import sys
|
||
|
|
import time
|
||
|
|
|
||
|
|
import portal
|
||
|
|
|
||
|
|
START_ROUND = 1
|
||
|
|
NUMBER_OF_ROUNDS = 10
|
||
|
|
NUMBER_OF_ARTICLES = 10
|
||
|
|
|
||
|
|
CONTINUE_FLAG = False
|
||
|
|
START_ID = '5052657'
|
||
|
|
STOP_ID = ''
|
||
|
|
|
||
|
|
SAVE_TO_PATH = 'PATH_TO_SAVE_FOLDER'
|
||
|
|
|
||
|
|
LOGIN_NAME = 'LOGIN'
|
||
|
|
LOGIN_PW = 'PASSWORD'
|
||
|
|
|
||
|
|
# MAIN START
|
||
|
|
|
||
|
|
curr_id = START_ID
|
||
|
|
|
||
|
|
print("Start time is", time.strftime("%H") + ":" + time.strftime("%M"))
|
||
|
|
|
||
|
|
chrome_options = portal.Options()
|
||
|
|
chrome_options.add_experimental_option("prefs", {"download.default_directory": SAVE_TO_PATH, "download.prompt_for_download": False, })
|
||
|
|
chrome_options.add_argument('--headless')
|
||
|
|
chrome = portal.WebBrowser()
|
||
|
|
|
||
|
|
for round_number in range(NUMBER_OF_ROUNDS):
|
||
|
|
if curr_id == '':
|
||
|
|
sys.exit(0)
|
||
|
|
print("*********************************")
|
||
|
|
print("** round #" + str(round_number + START_ROUND) + " of " + str(START_ROUND + NUMBER_OF_ROUNDS - 1))
|
||
|
|
print("** start time is", time.strftime("%H") + ":" + time.strftime("%M"))
|
||
|
|
print("*********************************")
|
||
|
|
|
||
|
|
chrome.start_chrome(chrome_options)
|
||
|
|
processor = portal.GBDownloader(chrome, SAVE_TO_PATH, CONTINUE_FLAG, suffix_seed = round_number + START_ROUND)
|
||
|
|
if not processor.login(LOGIN_NAME, LOGIN_PW):
|
||
|
|
sys.exit(0)
|
||
|
|
|
||
|
|
curr_id = processor.scan(curr_id, STOP_ID, NUMBER_OF_ARTICLES)
|
||
|
|
|
||
|
|
time.sleep(2)
|
||
|
|
del processor
|
||
|
|
|
||
|
|
chrome.stop_chrome()
|
||
|
|
time.sleep(3)
|
||
|
|
|
||
|
|
print("** end time is", time.strftime("%H") + ":" + time.strftime("%M"))
|
||
|
|
print("")
|
||
|
|
|
||
|
|
print("next start article ID = " + curr_id)
|
||
|
|
print("next START_ROUND = " + str(START_ROUND + NUMBER_OF_ROUNDS))
|