Init commit
This commit is contained in:
Jemacivan
38
backup/google_parser/README.md
Normal file
38
backup/google_parser/README.md
Normal file
@@ -0,0 +1,38 @@
|
||||
# Installing
|
||||
**Create VirtualEnv**
|
||||
```
|
||||
python -m venv venv
|
||||
```
|
||||
**Activate VirtualEnv**
|
||||
```
|
||||
source venv/bin/activate
|
||||
```
|
||||
**Installing requirements**
|
||||
```
|
||||
pip install -r requirements.txt
|
||||
```
|
||||
**Rename example_config.ini -> config.ini
|
||||
Enter Telegram Bot API Token on config.ini
|
||||
|
||||
Optional: setup webhook, telegram bot api server, etc.
|
||||
|
||||
Enter DocumentID ([How to get DocumentID](https://developers.google.com/docs/api/how-tos/overview#document_id))
|
||||
|
||||
[Create project and enable Docs API](https://developers.google.com/workspace/guides/create-project)
|
||||
|
||||
Put credentials.json ([How to get credentials.json](https://developers.google.com/workspace/guides/create-credentials?hl=ru#create_a_oauth_client_id_credential))**
|
||||
|
||||
**Run setup docs parser**
|
||||
```
|
||||
python setup_google_docs_api.py
|
||||
```
|
||||
**Follow Setup Wizard
|
||||
Open web page
|
||||
Login your account
|
||||
If you get error, setup redirect in Project**
|
||||
|
||||
# Running
|
||||
```
|
||||
python bot.py
|
||||
```
|
||||
**Run on docker: Supported!**
|
||||
31
backup/google_parser/example_config.ini
Normal file
31
backup/google_parser/example_config.ini
Normal file
@@ -0,0 +1,31 @@
|
||||
[Docs_Settings]
|
||||
Document_ID = 123ABC
|
||||
Config_folder = configs/
|
||||
token_file = token.json
|
||||
credentials_file = credentials.json
|
||||
data_file = data.json
|
||||
|
||||
[Bot]
|
||||
token = 123:JAKD
|
||||
; None = Not used local telegram bot api server
|
||||
; Example http://127.0.0.1:8888
|
||||
telegram_bot_api_server = none
|
||||
; True or False
|
||||
use_webhook = false
|
||||
ip = 127.0.0.1
|
||||
port = 3001
|
||||
; if you don`t use local TelegramBotAPI Server + WebHooks -> Set settings in bot.py
|
||||
|
||||
[Users]
|
||||
;Uncomment this variable, if you use filters to users
|
||||
;allowed_users = 0,1,2,3
|
||||
admin_users = 0,1,2,3
|
||||
|
||||
[DataBase]
|
||||
enable_logging = yes
|
||||
database_link = sqlite:///db.sqlite3
|
||||
|
||||
[announcements]
|
||||
;Seconds only/
|
||||
time = 14400
|
||||
;////////////
|
||||
2
backup/google_parser/parser/__init__.py
Normal file
2
backup/google_parser/parser/__init__.py
Normal file
@@ -0,0 +1,2 @@
|
||||
from .parser import get_about_replacements, docs_parse
|
||||
__all__ = ['get_about_replacements', 'docs_parse']
|
||||
108
backup/google_parser/parser/parser.py
Normal file
108
backup/google_parser/parser/parser.py
Normal file
@@ -0,0 +1,108 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
import os
|
||||
import json
|
||||
|
||||
from googleapiclient.discovery import build
|
||||
from google_auth_oauthlib.flow import InstalledAppFlow
|
||||
from google.auth.transport.requests import Request
|
||||
from google.oauth2.credentials import Credentials
|
||||
|
||||
from load import config
|
||||
from .utils import Helper
|
||||
|
||||
# If modifying these scopes, delete the file token.json.
|
||||
SCOPES = ['https://www.googleapis.com/auth/documents.readonly']
|
||||
|
||||
__all__ = ['docs_parse', 'get_about_replacements']
|
||||
|
||||
|
||||
def docs_parse() -> None:
|
||||
creds = None
|
||||
# The file token.json stores the user's access and refresh tokens, and is
|
||||
# created automatically when the authorization flow completes for the first
|
||||
# time.
|
||||
if os.path.exists(config.token_file):
|
||||
creds = Credentials.from_authorized_user_file(
|
||||
config.token_file,
|
||||
SCOPES
|
||||
)
|
||||
# If there are no (valid) credentials available, let the user log in.
|
||||
if not creds or not creds.valid:
|
||||
if creds and creds.expired and creds.refresh_token:
|
||||
creds.refresh(Request())
|
||||
else:
|
||||
flow = InstalledAppFlow.from_client_secrets_file(
|
||||
config.credentials_file, SCOPES)
|
||||
creds = flow.run_local_server(port=0)
|
||||
# Save the credentials for the next run
|
||||
with open(config.token_file, 'w') as token:
|
||||
token.write(creds.to_json())
|
||||
|
||||
service = build('docs', 'v1', credentials=creds)
|
||||
|
||||
# Retrieve the documents contents from the Docs service.
|
||||
document = service.documents().get(documentId=config.documentid).execute()
|
||||
if os.path.exists(config.data_file):
|
||||
os.remove(config.data_file)
|
||||
|
||||
with open(config.data_file, 'w') as f:
|
||||
json.dump(document, f, ensure_ascii=False)
|
||||
f.close()
|
||||
|
||||
|
||||
def read_parse_data():
|
||||
with open(config.data_file, 'r') as f:
|
||||
data = json.loads(f.read())
|
||||
f.close()
|
||||
return data
|
||||
|
||||
|
||||
def get_about_replacements() -> dict:
|
||||
helper = Helper()
|
||||
document = read_parse_data()
|
||||
info = []
|
||||
element = helper.get_table_element()
|
||||
|
||||
try:
|
||||
count = document['body']["content"][element]["table"]["rows"]
|
||||
except (IndexError, KeyError):
|
||||
element = helper.find_with_table(document)
|
||||
if element:
|
||||
count = document['body']["content"][element]["table"]["rows"]
|
||||
else:
|
||||
info = helper.find_with_text(document)
|
||||
|
||||
date = helper.get_date(document)
|
||||
|
||||
another_teacher = helper.teacher(document)
|
||||
|
||||
if element:
|
||||
for c in range(0, count):
|
||||
more_replaces = (document['body']
|
||||
["content"][element]["table"]
|
||||
["tableRows"][c]["tableCells"][1]
|
||||
["content"]
|
||||
)
|
||||
replaces = ''
|
||||
for i in range(0, len(more_replaces)):
|
||||
replaces += (document['body']["content"][element]["table"]
|
||||
["tableRows"][c]["tableCells"][1]
|
||||
["content"][i]["paragraph"]["elements"][0]
|
||||
["textRun"]["content"].rstrip("\n"))
|
||||
|
||||
info.append(
|
||||
(
|
||||
document['body']["content"][element]["table"]
|
||||
["tableRows"][c]["tableCells"][0]
|
||||
["content"][0]["paragraph"]["elements"][0]
|
||||
["textRun"]["content"].rstrip("\n"),
|
||||
replaces
|
||||
)
|
||||
)
|
||||
|
||||
return {
|
||||
'date': date if type(date) != type(False) else "Error" ,
|
||||
'data': dict(info),
|
||||
'another_teacher': another_teacher,
|
||||
}
|
||||
188
backup/google_parser/parser/utils.py
Normal file
188
backup/google_parser/parser/utils.py
Normal file
@@ -0,0 +1,188 @@
|
||||
import os
|
||||
import datetime
|
||||
from datetime import datetime as dt
|
||||
|
||||
from load import config
|
||||
|
||||
|
||||
def date_parser_helper(days:int, parse:str="%d.%m.20%y"):
|
||||
return dt.strftime(
|
||||
dt.now() +
|
||||
datetime.timedelta(days=days),
|
||||
parse
|
||||
)
|
||||
|
||||
'''
|
||||
self.months = {
|
||||
1: "січень",
|
||||
2: "лютий",
|
||||
3: "березень",
|
||||
4: "квітень",
|
||||
5: "травень",
|
||||
6: "червень",
|
||||
7: "липень",
|
||||
8: "серпень",
|
||||
9: "вересень",
|
||||
10: "жовтень",
|
||||
11: "листопад",
|
||||
12: "грудень"
|
||||
}
|
||||
'''
|
||||
|
||||
class Helper():
|
||||
|
||||
def __init__(self):
|
||||
self.date_now = date_parser_helper(0)
|
||||
self.date_next = date_parser_helper(1)
|
||||
self.weekend_pass = date_parser_helper(2)
|
||||
self.two_day_pass = date_parser_helper(3)
|
||||
|
||||
self.black_list = [
|
||||
'черговий викладач',
|
||||
self.date_now,
|
||||
self.date_next,
|
||||
self.weekend_pass,
|
||||
self.two_day_pass
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def find_with_table(document):
|
||||
c_element = 2
|
||||
while True:
|
||||
try:
|
||||
document['body']["content"][c_element]["table"]["rows"]
|
||||
break
|
||||
except KeyError:
|
||||
c_element += 1
|
||||
if c_element > 15:
|
||||
return False
|
||||
except IndexError:
|
||||
return False
|
||||
|
||||
with open("{}/table_element.txt".format(config.config_folder), 'w') as f:
|
||||
f.write(str(c_element))
|
||||
f.close()
|
||||
return c_element
|
||||
|
||||
def find_with_text(self, document):
|
||||
format_charset = '-'
|
||||
alternative_format_charset = "\t"
|
||||
element = 4
|
||||
data = []
|
||||
text = ''
|
||||
|
||||
while element < 15:
|
||||
doc = (
|
||||
document['body']["content"][element]
|
||||
["paragraph"]["elements"][0]["textRun"]["content"]
|
||||
).rstrip("\n").replace("–", "-", 1)
|
||||
if (
|
||||
(
|
||||
("-" in doc)
|
||||
#and
|
||||
#("\t" not in doc)
|
||||
)
|
||||
and
|
||||
([p not in doc.lower() for p in self.black_list][0])
|
||||
):
|
||||
try:
|
||||
group, text = doc.split(format_charset)
|
||||
except ValueError:
|
||||
if element > 6:
|
||||
break
|
||||
else:
|
||||
try:
|
||||
group, text = doc.split(alternative_format_charset)
|
||||
except ValueError:
|
||||
if element > 6:
|
||||
break
|
||||
if text != '':
|
||||
data.append(
|
||||
(group.strip(" "), text.lstrip(" ").replace("\t", ""))
|
||||
)
|
||||
element += 1
|
||||
return data
|
||||
|
||||
def get_date(self, document):
|
||||
date_element = 1
|
||||
while date_element < 16:
|
||||
try:
|
||||
date = (
|
||||
document['body']["content"][date_element]
|
||||
["paragraph"]["elements"][0]["textRun"]["content"]
|
||||
.rstrip(" \n"))
|
||||
except:
|
||||
date_element += 1
|
||||
if (
|
||||
(
|
||||
(
|
||||
self.date_now in date.lower()
|
||||
.lstrip("заміни").lstrip("на").replace(" ", "")
|
||||
)
|
||||
or
|
||||
(
|
||||
self.date_next in date.lower()
|
||||
.lstrip("заміни").lstrip("на").replace(" ", "")
|
||||
)
|
||||
or
|
||||
(
|
||||
self.weekend_pass in date.lower()
|
||||
.lstrip("заміни").lstrip("на").replace(" ", "")
|
||||
)
|
||||
or
|
||||
(
|
||||
self.two_day_pass in date.lower()
|
||||
.lstrip("заміни").lstrip("на").replace(" ", "")
|
||||
)
|
||||
)
|
||||
or
|
||||
(
|
||||
"заміни на" in date.lower()
|
||||
)
|
||||
):
|
||||
return date
|
||||
else:
|
||||
date_element += 1
|
||||
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def get_table_element():
|
||||
if os.path.exists(f"{config.config_folder}/table_element.txt"):
|
||||
element = int(
|
||||
open(
|
||||
f"{config.config_folder}/table_element.txt",
|
||||
'r'
|
||||
)
|
||||
.read()
|
||||
)
|
||||
else:
|
||||
element = 6
|
||||
return element
|
||||
|
||||
@staticmethod
|
||||
def teacher(document):
|
||||
element = 1
|
||||
while element < 6:
|
||||
if "paragraph" in document['body']["content"][element]:
|
||||
length_element = (len(document['body']["content"][element]
|
||||
["paragraph"]["elements"]))
|
||||
|
||||
doc = (
|
||||
document['body']["content"][element]["paragraph"]["elements"]
|
||||
[0]["textRun"]["content"].rstrip("\n")
|
||||
)
|
||||
if 'черговий викладач' in doc.lower().replace("–", ""):
|
||||
return doc
|
||||
|
||||
elif length_element > 1:
|
||||
for p in range(length_element):
|
||||
doc = (
|
||||
document['body']["content"][element]
|
||||
["paragraph"]["elements"]
|
||||
[p]["textRun"]["content"].rstrip("\n")
|
||||
)
|
||||
if 'черговий викладач' in doc.lower().replace("–", ""):
|
||||
return doc
|
||||
|
||||
element += 1
|
||||
9
backup/google_parser/requirements.txt
Normal file
9
backup/google_parser/requirements.txt
Normal file
@@ -0,0 +1,9 @@
|
||||
google-api-python-client
|
||||
google-auth-httplib2
|
||||
google-auth-oauthlib
|
||||
peewee
|
||||
aiogram
|
||||
cryptography
|
||||
pymysqldb
|
||||
psycopg2
|
||||
aioschedule
|
||||
9
backup/google_parser/setup_google_docs_api.py
Normal file
9
backup/google_parser/setup_google_docs_api.py
Normal file
@@ -0,0 +1,9 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
'''
|
||||
Don`t move this file!
|
||||
'''
|
||||
|
||||
if __name__ == '__main__':
|
||||
from parser import docs_parse
|
||||
docs_parse()
|
||||
Reference in New Issue
Block a user