mirror of
https://github.com/TeamWiseFlow/wiseflow.git
synced 2025-01-23 10:50:25 +08:00
33 lines
876 B
Python
33 lines
876 B
Python
import asyncio
|
|
from insights import pipeline, pb, logger
|
|
|
|
counter = 1
|
|
|
|
|
|
async def process_site(site, counter):
|
|
if not site['per_hours'] or not site['url']:
|
|
return
|
|
if counter % site['per_hours'] == 0:
|
|
logger.info(f"applying {site['url']}")
|
|
await pipeline(site['url'].rstrip('/'))
|
|
|
|
|
|
async def schedule_pipeline(interval):
|
|
global counter
|
|
while True:
|
|
sites = pb.read('sites', filter='activated=True')
|
|
logger.info(f'task execute loop {counter}')
|
|
await asyncio.gather(*[process_site(site, counter) for site in sites])
|
|
|
|
counter += 1
|
|
logger.info(f'task execute loop finished, work after {interval} seconds')
|
|
await asyncio.sleep(interval)
|
|
|
|
|
|
async def main():
|
|
interval_hours = 1
|
|
interval_seconds = interval_hours * 60 * 60
|
|
await schedule_pipeline(interval_seconds)
|
|
|
|
asyncio.run(main())
|