Spam words moved to DB. Add command: /add_spam

2024-05-30 22:42:43 +03:00
parent 1a23d576ed
commit 034c05d957
14 changed files with 43 additions and 287 deletions
@@ -1,110 +0,0 @@
-# A generic, single database configuration.
-
-[alembic]
-# path to migration scripts
-script_location = alembic
-
-# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
-# Uncomment the line below if you want the files to be prepended with date and time
-# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file
-# for all available tokens
-# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
-
-# sys.path path, will be prepended to sys.path if present.
-# defaults to the current working directory.
-prepend_sys_path = .
-
-# timezone to use when rendering the date within the migration file
-# as well as the filename.
-# If specified, requires the python-dateutil library that can be
-# installed by adding `alembic[tz]` to the pip requirements
-# string value is passed to dateutil.tz.gettz()
-# leave blank for localtime
-# timezone =
-
-# max length of characters to apply to the
-# "slug" field
-# truncate_slug_length = 40
-
-# set to 'true' to run the environment during
-# the 'revision' command, regardless of autogenerate
-# revision_environment = false
-
-# set to 'true' to allow .pyc and .pyo files without
-# a source .py file to be detected as revisions in the
-# versions/ directory
-# sourceless = false
-
-# version location specification; This defaults
-# to alembic/versions.  When using multiple version
-# directories, initial revisions must be specified with --version-path.
-# The path separator used here should be the separator specified by "version_path_separator" below.
-# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions
-
-# version path separator; As mentioned above, this is the character used to split
-# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep.
-# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas.
-# Valid values for version_path_separator are:
-#
-# version_path_separator = :
-# version_path_separator = ;
-# version_path_separator = space
-version_path_separator = os  # Use os.pathsep. Default configuration used for new projects.
-
-# set to 'true' to search source files recursively
-# in each "version_locations" directory
-# new in Alembic version 1.10
-# recursive_version_locations = false
-
-# the output encoding used when revision files
-# are written from script.py.mako
-# output_encoding = utf-8
-
-sqlalchemy.url = sqlite:///E:\Projects\PythonChatRu\LutzPyBot\app\db\db.sqlite
-
-
-[post_write_hooks]
-# post_write_hooks defines scripts or Python functions that are run
-# on newly generated revision scripts.  See the documentation for further
-# detail and examples
-
-# format using "black" - use the console_scripts runner, against the "black" entrypoint
-# hooks = black
-# black.type = console_scripts
-# black.entrypoint = black
-# black.options = -l 79 REVISION_SCRIPT_FILENAME
-
-# Logging configuration
-[loggers]
-keys = root,sqlalchemy,alembic
-
-[handlers]
-keys = console
-
-[formatters]
-keys = generic
-
-[logger_root]
-level = WARN
-handlers = console
-qualname =
-
-[logger_sqlalchemy]
-level = WARN
-handlers =
-qualname = sqlalchemy.engine
-
-[logger_alembic]
-level = INFO
-handlers =
-qualname = alembic
-
-[handler_console]
-class = StreamHandler
-args = (sys.stderr,)
-level = NOTSET
-formatter = generic
-
-[formatter_generic]
-format = %(levelname)-5.5s [%(name)s] %(message)s
-datefmt = %H:%M:%S
@@ -1 +0,0 @@
-Generic single-database configuration.
@@ -1,78 +0,0 @@
-from logging.config import fileConfig
-
-from sqlalchemy import engine_from_config
-from sqlalchemy import pool
-
-from alembic import context
-
-# this is the Alembic Config object, which provides
-# access to the values within the .ini file in use.
-config = context.config
-
-# Interpret the config file for Python logging.
-# This line sets up loggers basically.
-if config.config_file_name is not None:
-    fileConfig(config.config_file_name)
-
-# add your model's MetaData object here
-# for 'autogenerate' support
-# from myapp import mymodel
-# target_metadata = mymodel.Base.metadata
-target_metadata = None
-
-# other values from the config, defined by the needs of env.py,
-# can be acquired:
-# my_important_option = config.get_main_option("my_important_option")
-# ... etc.
-
-
-def run_migrations_offline() -> None:
-    """Run migrations in 'offline' mode.
-
-    This configures the context with just a URL
-    and not an Engine, though an Engine is acceptable
-    here as well.  By skipping the Engine creation
-    we don't even need a DBAPI to be available.
-
-    Calls to context.execute() here emit the given string to the
-    script output.
-
-    """
-    url = config.get_main_option("sqlalchemy.url")
-    context.configure(
-        url=url,
-        target_metadata=target_metadata,
-        literal_binds=True,
-        dialect_opts={"paramstyle": "named"},
-    )
-
-    with context.begin_transaction():
-        context.run_migrations()
-
-
-def run_migrations_online() -> None:
-    """Run migrations in 'online' mode.
-
-    In this scenario we need to create an Engine
-    and associate a connection with the context.
-
-    """
-    connectable = engine_from_config(
-        config.get_section(config.config_ini_section, {}),
-        prefix="sqlalchemy.",
-        poolclass=pool.NullPool,
-    )
-
-    with connectable.connect() as connection:
-        context.configure(
-            connection=connection, target_metadata=target_metadata
-        )
-
-        with context.begin_transaction():
-            context.run_migrations()
-
-
-if context.is_offline_mode():
-    run_migrations_offline()
-else:
-    run_migrations_online()
@@ -1,24 +0,0 @@
-"""${message}
-
-Revision ID: ${up_revision}
-Revises: ${down_revision | comma,n}
-Create Date: ${create_date}
-
-"""
-from alembic import op
-import sqlalchemy as sa
-${imports if imports else ""}
-
-# revision identifiers, used by Alembic.
-revision = ${repr(up_revision)}
-down_revision = ${repr(down_revision)}
-branch_labels = ${repr(branch_labels)}
-depends_on = ${repr(depends_on)}
-
-
-def upgrade() -> None:
-    ${upgrades if upgrades else "pass"}
-
-
-def downgrade() -> None:
-    ${downgrades if downgrades else "pass"}
@@ -1,24 +0,0 @@
-"""add permissions
-
-Revision ID: 1b1f99b95b38
-Revises: 9d160884887c
-Create Date: 2023-07-30 18:33:35.400071
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = '1b1f99b95b38'
-down_revision = '9d160884887c'
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    pass
-
-
-def downgrade() -> None:
-    pass
@@ -1,24 +0,0 @@
-"""My migration message
-
-Revision ID: 9d160884887c
-Revises: 
-Create Date: 2023-04-06 00:47:17.551695
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = '9d160884887c'
-down_revision = None
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    pass
-
-
-def downgrade() -> None:
-    pass
@@ -1 +1 @@
-from . import set_commands
+# from . import set_commands  # Временное решение. Не коммитить раскомментированным - упадут тесты.
@@ -49,23 +49,6 @@ RULES = types.InlineKeyboardButton("👨🏼‍🎓 Правила чата", ur
 FAQ = types.InlineKeyboardButton("❔ Частые вопросы", url="https://telegra.ph/faq-10-07-4")
 LIB = types.InlineKeyboardButton("📚 Книги", url="https://telegra.ph/what-to-read-10-06")

-SPAM = {"me.sv/", "tg.sv/", "goo.by/", "go.sv/", "intim.video/", "uclck.ru/",
-        '💎', '🔞', '➡️', '💰', '💸', '‼️',
-        'раб0т', '0плат', 'новый способ заработка', 'криптовалют', 'цифровых валют', 'Binance', 'ByBit',
-        'Помогу заработать', 'арбитраж', 'DEX бирж', 'любую девушку', 'прибыль у вaс', 'обучaю быстро',
-        'Предлагаю новый вид заработка', 'межбиржев', 'КУПЮРЫ', 'в неделю выходит до', 'soax',
-        'предложить заработок', 'голые фот', 'интимные фот', 'заработка в крипте', 'Веду набор', 'Идёт набор',
-        'Доход от', 'Trading', 'P2P', 'Идет набор', '$ в неделю', 'обучаю с нуля', 'доходы для всех',
-        'по трейдингу', 'фальшивые рубли', 'зароботку', 'Обучу вас', 'остались курсы', 'Crypt',
-        'Ищу партнеров', 'ищем партнеров', 'в поиске партнеров', 'обучим бесплатно', 'обучу бесплатно',
-        'обучим бесплатно', 'обучаю бесплатно', 'цифровых активов', 'цифровые активы', 'цифровые валюты',
-        'никаких предоплат', 'предоплаты не требуется', 'без предоплат', 'предоплата не требуется',
-        'предоплата не нужна', 'можете ознакомиться', 'хочу пригласить Вас', 'сотрудничество на удаленке',
-        'Только совершеннолетние', 'хороший заработок',
-
-        'в лc', "в личные сообще", 'напиши в личку', 'Писать в ЛИЧКУ', 'Пишите в личку', 'в л.с.', 'в лс',
-        'Писать в личные', 'в личку', 'в личке', 'в личных сооб', 'пишите в личные', 'пишем в личные', 'Плюс в личные'
-        }

 NON_GRATA = {"дудар", "хауди", "dudar"}
 BAN_WORDS = {"GREEN"}
@@ -1,5 +1,4 @@
 """ Send User's info. """
-
 from urllib import parse
 from urllib.error import URLError
 from urllib.request import urlopen
@@ -7,8 +6,9 @@ from urllib.request import urlopen
 from telebot import types

 from .constants import (
-    SPAM, BAN_WORDS, NON_GRATA, ADMIN_ID, ONLY_RUS_LETTERS, ONLY_ENG_LETTERS, RULES_TEXT, SYMBOLS, URL_RX
+    NON_GRATA, ADMIN_ID, ONLY_RUS_LETTERS, ONLY_ENG_LETTERS, RULES_TEXT, SYMBOLS, BAN_WORDS
 )
+from .models import session, BadWord

 WARNS: dict[int, int] = {}

@@ -40,7 +40,7 @@ def is_spam(message_text: str) -> bool:
    """
    if is_mixed(message_text):
        return True
-    return any(text.casefold() in message_text.casefold() for text in SPAM)
+    return any(bad_word.word.casefold() in message_text.casefold() for bad_word in session.query(BadWord).all())


 def is_mixed(text: str) -> bool:
@@ -54,6 +54,15 @@ class Chat(Base):
        return f'Chat({self.chat_id=}, {self.title=})'


+class BadWord(Base):
+    __tablename__ = 'bad_words'
+    id = Column(Integer, primary_key=True)
+    word = Column(String)
+
+    def __repr__(self):
+        return f'BadWord({self.id}: {self.word=})'
+
+
 CACHE: dict[int, Type[Chat]] = {}


@@ -95,6 +104,12 @@ def add_quote(chat_id: int, text: str):
    _reload_chat(chat_id)


+def add_spam(chat_id: int, text: str):
+    session.add(BadWord(word=text))
+    session.commit()
+    _reload_chat(chat_id)
+
+
 def chat_by_id(chat_id: int) -> None | Chat:
    return CACHE.get(chat_id)

@@ -26,6 +26,7 @@ for white_id in WHITEIDS:
            telebot.types.BotCommand("links", "/ссылки на правила, чзв и библиотеку"),
            telebot.types.BotCommand("tsya", "/тся и /ться"),
            telebot.types.BotCommand("add", "добавить цитату [Whitelist]"),
+            telebot.types.BotCommand("add_spam", "добавить в спам [Whitelist]"),
            telebot.types.BotCommand("remove", "удалить цитату [Whitelist]"),
        ],
        scope=telebot.types.BotCommandScopeChatMember(PYTHONCHATRU, white_id)
@@ -9,7 +9,8 @@ from telebot import types, util
 from .constants import LOG_COMM
 from .filters import is_white_id
 from .helpers import is_nongrata, short_user_data, has_links
-from .models import all_chat_quotes, add_quote, is_quote_in_chat, delete_quote_in_chat, CACHE
+from .models import all_chat_quotes, add_quote, is_quote_in_chat, delete_quote_in_chat, CACHE, add_spam, session, \
+    BadWord
 from .robot import bot

 TYPING_TIMEOUT = 0.13 / 4  # Reading time is quarter of the same text writing time
@@ -72,6 +73,22 @@ async def add_stan_quote(message: types.Message):
            await bot.send_message(message.chat.id, f"⛔️ Не добавил, есть токое: {quote}", parse_mode='Markdown')


+@bot.message_handler(func=is_white_id, commands=["add_spam"])
+async def add_spam_handler(message: types.Message):
+    if message.reply_to_message and message.reply_to_message.text:
+        quote = (message.quote and message.quote.text) or message.reply_to_message.text
+        logging.error(f'{session.query(BadWord).filter_by(word=quote).first()}')
+        if not session.query(BadWord).filter_by(word=quote).first():
+            add_spam(message.chat.id, quote)
+            ok_message = await bot.send_message(message.chat.id, f"⭐️ Добавил в спам: {quote}", parse_mode='Markdown')
+            await bot.delete_message(message.chat.id, message.id)
+            await asyncio.sleep(3)
+            await bot.delete_message(message.chat.id, ok_message.id)
+            logging.info(LOG_COMM % (message.chat.title, message.from_user.id, message.from_user.full_name, f'[SPAM] {message.reply_to_message.text}'))
+        else:
+            await bot.send_message(message.chat.id, f"⛔️ Не добавил, есть токое: {quote}", parse_mode='Markdown')
+
+
@bot.message_handler(func=is_white_id, commands=["remove"])
 async def remove_stan_quote(message: types.Message):
    if message.reply_to_message and message.reply_to_message.text:
@@ -1,10 +1,11 @@
 from unittest import TestCase, main
 from unittest.mock import MagicMock
-from src.constants import SPAM, BAN_WORDS, ADMIN_ID, RULES_TEXT
+from src.constants import BAN_WORDS, ADMIN_ID, RULES_TEXT
 from src.helpers import (
    is_spam, is_mixed, is_ban_words_in_caption, is_in_not_allowed, is_nongrata, is_admin, fetch_rule, cleaned_text,
    remove_spaces, has_no_letters, has_links
 )
+from src.models import BadWord, session

 REAL_CASES = (
    "Оформим резидeнтствo ОАЭ без предoплаты за 5000 дирхам. Пoдробнoсти в лc",
@@ -48,9 +49,9 @@ TEST_CASES = ('here it me.sv/122', "My site is tg.sv/home", "look here goo.by/ho

 class TestSpam(TestCase):
    def test_simple(self):
-        for spam_word in SPAM:
-            with self.subTest(f"check spam word({spam_word})"):
-                self.assertTrue(is_spam(spam_word))
+        for spam_word in session.query(BadWord).all():
+            with self.subTest(f"check spam word({spam_word.word})"):
+                self.assertTrue(is_spam(spam_word.word))

    def test_case_first(self):
        for text in REAL_CASES: