Merge branch 'master' of github.com:olinpin/us-disclosures

.env.example

@@ -1,5 +1,5 @@
 TELEGRAM_API_KEY=
 TELEGRAM_CHANNEL_ID=
-DB_PATH=
+DB_PATH=test.db
-SCHEMA_PATH=
+SCHEMA_PATH=schema.sql
-SEED_PATH=
+SEED_PATH=seed.sql

README.md

@@ -0,0 +1,41 @@
+# US Disclosures
+Simple python app to track US Disclosurs forms, where members of congress post their investments. Script checks if there are any new disclosures for specified members and if there are, those disclosures will be sent as a telegram message. Data is from [https://disclosures-clerk.house.gov/FinancialDisclosure](https://disclosures-clerk.house.gov/FinancialDisclosure)
+
+## How to run
+1. Clone the repo 
+```bash
+git clone https://github.com/olinpin/us-disclosures
+```
+2. Change directory into the repo 
+```bash
+cd us-disclosures
+```
+3. Copy .env file 
+```bash
+cp .env.example .env
+```
+4. Fill .env file with using your favorite editor:
+```bash
+vim .env
+```
+5. Create virtual environment and activate it 
+```bash
+python -m venv venv && source venv/bin/activate
+```
+6. Install dependencies 
+```bash
+pip install -r requirements.txt
+```
+7. Edit the `seed.sql` file with names you'd like to track (as seen in the example in the file)
+8. Run the script 
+```bash
+python disclosures/financial_disclosure.py
+```
+9. Receive telegram messages with disclosures (for best results I recommend running this every 30 minutes with crontab)
+
+### .env variables
+- `TELEGRAM_API_KEY` - can be obtained through the official [telegram docs](https://github.com/olinpin/us-disclosures)
+- `TELEGRAM_CHANNEL_ID` - that's the channel ID of your telegram bot that you created according to docs above (can be a CSV)
+- `DB_PATH` - can stay the same
+- `SCHEMA_PATH` - can stay the same
+- `SEED_PATH` - can stay the same

disclosures/financial_disclosure.py

@@ -1,8 +1,8 @@
 import requests
 from bs4 import BeautifulSoup
 import os
-from telegram_bot import Telegram
+from util.telegram_bot import Telegram
-from db import DB
+from util.db import DB
 from dotenv import load_dotenv
 import asyncio
 import datetime
@@ -12,16 +12,20 @@ load_dotenv()
 
 class Disclosures:
     def __init__(
-        self, telegram_api_key, telegram_channel, db_name, schema_path, seed_path
+        self, telegram_api_key, telegram_channels, db_name, schema_path, seed_path
     ):
-        self.telegram = Telegram(telegram_api_key, telegram_channel)
+        self.telegrams = []
+        for telegram_channel in telegram_channels.split(","):
+            self.telegrams.append(Telegram(telegram_api_key, telegram_channel))
         self.db = DB(db_name, schema_path, seed_path)
 
     async def send_message(self, message, return_value=True):
         try:
-            await self.telegram.send_message(message)
+            for telegram in self.telegrams:
+                await telegram.send_message(message)
             return return_value
         except Exception as e:
+            self.log(f"Error sending message: {e}, message: {message}, return_value: {return_value}")
             return False
 
     def getDocuments(self, name="pelosi"):
@@ -92,7 +96,6 @@ class Disclosures:
             documents[id] = self.getDocuments(name)
         values = self.prepareValues(documents)
         tasks = []
-        sent = []
         for v in values:
             message = f"New disclosure from {[name for id, name in members if id == v[0]][0]} for the year {v[1]}. {v[2]} {v[3]}"
             tasks.append(asyncio.create_task(self.send_message(message, v)))
@@ -101,12 +104,15 @@ class Disclosures:
         results = await asyncio.gather(*tasks)
         sent = [r for r in results if r]
 
-        self.insertDisclosures(sent)
+        self.insertDisclosures(values)
 
-        print(
+        self.log(
-            f"{datetime.datetime.now()} - Sent {len(sent)} disclosures. Inserted {len(sent)} disclosures. Total disclosures {len(values)}"
+            f"Sent {len(sent)} disclosures. Inserted {len(values)} disclosures. Total disclosures {len(values)}"
         )
 
+    def log(self, message):
+        print(f"{datetime.datetime.now()} - {message}")
+
 
 d = Disclosures(
     os.getenv("TELEGRAM_API_KEY"),
@@ -115,4 +121,7 @@ d = Disclosures(
     os.getenv("SCHEMA_PATH"),
     os.getenv("SEED_PATH"),
 )
-asyncio.run(d.run())
+try:
+    asyncio.run(d.run())
+except Exception as e:
+    d.log(f"Error while running: {e}")