aboutsummaryrefslogtreecommitdiff
path: root/cgi/formatting.py
blob: cb512abeb8bd2e7a53fb6bebeb312c5d58d9c6de (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
# coding=utf-8
import string
import html
import os
import re
import time

from database import *
from framework import *
from post import regenerateAccess
#from xhtml_clean import Cleaner

from settings import Settings


def format_post(message, ip, parentid, parent_timestamp=0):
    """
    Formats posts using the specified format
    """
    board = Settings._.BOARD
    using_markdown = False

    # Escape any HTML if user is not using Markdown or HTML
    if not Settings.USE_HTML:
        message = html.escape(message)

    # Strip text
    message = message.rstrip()[0:8000]

    # Treat HTML
    if Settings.USE_MARKDOWN:
        message = markdown(message)
        using_markdown = True
    if Settings.USE_HTML:
        message = onlyAllowedHTML(message)

    if Settings.VIDEO_THUMBS:
        (message, affected) = videoThumbs(message)
        # if affected:
        #  message = close_html(message)

    message = clickableURLs(message)
    message = checkRefLinks(message, parentid, parent_timestamp)
    message = checkWordfilters(message, ip, board["dir"])

    # If not using markdown quotes must be created and \n changed for HTML line breaks
    if not using_markdown:
        message = re.compile(r"^[\s\t]*(\n)+").sub("", message)
        message = re.compile(r"\n(\n)+").sub("\n\n", message)
        message = checkQuotes(message)
        message = message.replace("\n", "<br />")

    return message


def tripcode(name):
    """
    Calculate tripcode to match output of most imageboards
    """
    if name == '':
        return '', ''

    board = Settings._.BOARD
    key = Settings.TRIP_CHAR

    # if there's a trip
    (namepart, marker, trippart) = name.partition('#')
    if marker:
        namepart = cleanString(namepart)
        trip = ''

        # secure tripcode
        if Settings.ALLOW_SECURE_TRIPCODES and '#' in trippart:
            (trippart, securemarker, securepart) = trippart.partition('#')
            try:
                securepart = securepart.encode("sjis", "ignore")
            except:
                pass

            # encode secure tripcode
            trip = getMD5(securepart + Settings.SECRET)
            trip = trip.encode('base64').replace('\n', '')
            trip = trip.encode('rot13')
            trip = key+key+trip[2:12]

            # return it if we don't have a normal tripcode
            if trippart == '':
                return namepart, trip

        # do normal tripcode
        from crypt import crypt
        #try:
        #    trippart = trippart.encode("sjis", "ignore")
        #except:
        #    pass

        trippart = html.unescape(trippart)
        trippart = html.escape(trippart, True)
        salt = re.sub(r"[^\.-z]", ".", (trippart + "H..")[1:3])
        salt = salt.translate(str.maketrans(r":;=?@[\]^_`", "ABDFGabcdef"))
        trip = key + crypt(trippart, salt)[-10:] + trip

        return namepart, trip

    return name, ''


def iphash(ip, post, t, useid, mobile, agent, cap_id, hide_end, has_countrycode):
    current_t = time.time()

    if cap_id:
        id = cap_id
    elif post['email'] and useid == 1:
        id = '???'
    elif ip == "127.0.0.1":
        id = '???'
    else:
        day = int((current_t + (Settings.TIME_ZONE*3600)) / 86400)
        word = ',' + str(day)  # IDs change every 24 hours
        word += ',' + str(t)  # IDs vary depending on thread
        
        id = getb64(getMD5(ip + word + Settings.SECRET))[-10:-2]

    if hide_end:
        id += '*'
    elif addressIsTor(ip):
        id += 'T'
    elif 'Dalvik' in agent:
        id += 'R'
    elif 'Android' in agent:
        id += 'a'
    elif 'iPhone' in agent:
        id += 'i'
    elif useid == 3:
        if 'Firefox' in agent:
            id += 'F'
        elif 'Safari' in agent and not 'Chrome' in agent:
            id += 's'
        elif 'Chrome' in agent:
            id += 'C'
        elif 'SeaMonkey' in agent:
            id += 'S'
        elif 'Edge' in agent:
            id += 'E'
        elif 'Opera' in agent or 'OPR' in agent:
            id += 'o'
        elif 'MSIE' in agent or 'Trident' in agent:
            id += 'I'
        elif mobile:
            id += 'Q'
        else:
            id += '0'
    elif mobile:
        id += 'Q'
    else:
        id += '0'

    if addressIsBanned(ip, ""):
        id += '#'
    elif (not has_countrycode and not addressIsTor(ip) and
          (addressIsProxy(ip) or not addressIsES(ip))):
        id += '!'
    
    logging.info("{} {} {}".format(ip, agent, id))

    return id


def cleanString(string, escape=True, quote=False):
    string = string.strip()
    if escape:
        string = html.escape(string, quote)
    return string


def clickableURLs(message):
    # URL
    message = re.compile(r'( |^|:|\(|\[)((?:https?://|ftp://|mailto:|news:|irc:)[^\s<>()"]*?(?:\([^\s<>()"]*?\)[^\s<>()"]*?)*)((?:\s|<|>|"|\.|\|\]|!|\?|,|&#44;|&quot;)*(?:[\s<>()"]|$))',
                         re.M).sub(r'\1<a href="\2" rel="nofollow" target="_blank">\2</a>\3', message)
    # Emails
    message = re.compile(r"( |^|:)([A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,6})", re.I | re.M).sub(
        r'\1<a href="mailto:\2" rel="nofollow">&lt;\2&gt;</a>', message)

    return message


def videoThumbs(message):
    # Youtube
    __RE = re.compile(
        r"^(?: +)?(https?://(?:www\.)?youtu(?:be\.com/watch\?v=|\.be/)([\w\-]+))(?: +)?$", re.M)
    matches = __RE.finditer(message)
    if matches:
        import json
        import urllib.request, urllib.parse, urllib.error

        v_ids = []
        videos = {}

        for match in matches:
            v_id = match.group(2)
            if v_id not in v_ids:
                v_ids.append(v_id)
                videos[v_id] = {
                    'span': match.span(0),
                    'url':  match.group(1),
                }
            if len(v_ids) > Settings.VIDEO_THUMBS_LIMIT:
                raise UserError("Has incluído muchos videos en tu mensaje. El máximo es %d." % Settings.VIDEO_THUMBS_LIMIT)

    if videos:
        params = {
            'key': Settings.GOOGLE_API_KEY,
            'part': 'snippet,contentDetails',
            'id': ','.join(v_ids)
        }
        r_url = "https://www.googleapis.com/youtube/v3/videos?" + \
            urllib.parse.urlencode(params)
        res = urllib.request.urlopen(r_url)
        res_json = json.load(res)

        offset = 0
        for item in res_json['items']:
            v_id = item['id']
            (start, end) = videos[v_id]['span']
            end += 1  # remove endline

            try:
                new_url = '<a href="%(url)s" target="_blank" class="yt"><span class="pvw"><img src="%(thumb)s" /></span><b>%(title)s</b> (%(secs)s)<br />%(channel)s</a><br />' \
                    % {'title': item['snippet']['title'],
                       'channel': item['snippet']['channelTitle'],
                       'secs': parseIsoPeriod(item['contentDetails']['duration']),
                       'url': videos[v_id]['url'],
                       'id': v_id,
                       'thumb': item['snippet']['thumbnails']['default']['url'], }
            except UnicodeDecodeError:
                raise UserError(repr(v_id))
            message = message[:start+offset] + new_url + message[end+offset:]
            offset += len(new_url) - (end-start)

    return (message, len(videos))


def fixMobileLinks(message):
    """
    Shorten long links; Convert >># links into a mobile version
    """
    board = Settings._.BOARD

    # If textboard
    if board["board_type"] == 1:
        message = re.compile(r'<a href="/(\w+)/read/(\d+)(\.html)?/*(.+)"').sub(
            r'<a href="/cgi/mobileread/\1/\2/\4"', message)
    else:
        message = re.compile(r'<a href="/(\w+)/res/(\d+)\.html#(\d+)"').sub(
            r'<a href="/cgi/mobileread/\1/\2#\3"', message)

    return message


def checkRefLinks(message, parentid, parent_timestamp):
    """
    Check for >># links in posts and replace with the HTML to make them clickable
    """
    board = Settings._.BOARD

    if board["board_type"] == 1:
        # Textboard
        if parentid:
            message = re.compile(r'&gt;&gt;(\d+(,\d+|-(?=[ \d\n])|\d+)*n?)').sub(
                '<a href="' + Settings.BOARDS_URL + board['dir'] + '/read/' + str(parent_timestamp) + r'/\1">&gt;&gt;\1</a>', message)
    else:
        # Imageboard
        quotes_id_array = re.findall(r"&gt;&gt;([0-9]+)", message)
        for quotes in quotes_id_array:
            try:
                post = FetchOne('SELECT * FROM `posts` WHERE `id` = %s AND `boardid` = %s LIMIT 1',
                                (quotes, board['id']))
                if post['parentid']:
                    message = re.compile("&gt;&gt;" + quotes).sub('<a href="' + Settings.BOARDS_URL +
                                                                  board['dir'] + '/res/' + str(post['parentid']) + '.html#' + quotes + '">&gt;&gt;' + quotes + '</a>', message)
                else:
                    message = re.compile("&gt;&gt;" + quotes).sub('<a href="' + Settings.BOARDS_URL +
                                                                  board['dir'] + '/res/' + str(post['id']) + '.html#' + quotes + '">&gt;&gt;' + quotes + '</a>', message)
            except:
                message = re.compile(
                    "&gt;&gt;" + quotes).sub(r'<span class="q">&gt;&gt;'+quotes+'</span>', message)

    return message


def checkQuotes(message):
    """
    Check for >text in posts and add span around it to color according to the css
    """
    message = re.compile(
        r"^&gt;(.*)$", re.MULTILINE).sub(r'<span class="q">&gt;\1</span>', message)
    return message


def escapeHTML(string):
    string = string.replace('<', '&lt;')
    string = string.replace('>', '&gt;')
    return string


def onlyAllowedHTML(message):
    """
    Allow <b>, <i>, <u>, <strike>, and <pre> in posts, along with the special <aa>
    """
    message = sanitize_html(message)
    #message = re.compile(r"\[aa\](.+?)\[/aa\]", re.DOTALL | re.IGNORECASE).sub("<span class=\"sjis\">\\1</span>", message)

    return message


def close_html(message):
    """
    Old retarded version of sanitize_html, it just closes open tags.
    """
    #import BeautifulSoup

    #message = message.encode('utf-8')
    #soup = BeautifulSoup.BeautifulSoup(message)

    #return str(soup).replace('&#13;', '').encode('utf-8')

    try:
      l = message.rindex('<')
    except ValueError:
      return message
    test = message[l:]
    try:
      r = test.rindex('>')
      return message
    except ValueError:
      return message[:l]


def remove_html(message):
    string = message.replace("<br />", "\n")
    string = re.sub(r"<.*?>", '', string)
    string = string.replace("\n", "<br />")
    return string


def sanitize_html(message, decode=True):
    """
    Clean the code and allow only a few safe tags.
    """
    import BeautifulSoup

    # Decode message from utf-8 if required
    if decode:
        message = message.decode('utf-8', 'replace')

    # Create HTML Cleaner with our allowed tags
    whitelist_tags = ["a", "b", "br", "blink", "code", "del", "em",
                      "i", "marquee", "root", "strike", "strong", "sub", "sup", "u"]
    whitelist_attr = ["href"]

    soup = BeautifulSoup.BeautifulSoup(message)

    # Remove tags that aren't allowed
    for tag in soup.findAll():
        if not tag.name.lower() in whitelist_tags:
            tag.name = "span"
            tag.attrs = []
        else:
            for attr in [attr for attr in tag.attrs if attr not in whitelist_attr]:
                del tag[attr]

    # We export the soup into a correct XHTML string
    string = str(soup).encode('utf-8')
    # We remove some anomalies we don't want
    string = string.replace('<br/>', '<br />').replace('&#13;', '')

    return string


def markdown(message):
    import markdown
    if message.strip() != "":
        # return markdown.markdown(message).rstrip("\n").rstrip("<br />")
        return markdown.markdown(message, extras=["cuddled-lists", "code-friendly"]).encode('utf-8')
    else:
        return ""


def checkWordfilters(message, ip, board):
    wordfilters = FetchAll(
        "SELECT * FROM `filters` WHERE `type` = '0' ORDER BY `id` ASC")
    for wordfilter in wordfilters:
        if wordfilter["boards"]:
            boards = str2boards(wordfilter["boards"])
        if not wordfilter["boards"] or board in boards:
            if wordfilter['action'] == 0:
                if not re.search(wordfilter['from'], message, re.DOTALL | re.IGNORECASE) is None:
                    raise UserError(wordfilter['reason'])
            elif wordfilter['action'] == 1:
                message = re.compile(wordfilter['from'], re.DOTALL | re.IGNORECASE).sub(
                    wordfilter['to'], message)
            elif wordfilter['action'] == 2:
                # Ban
                if not re.search(wordfilter['from'], message, re.DOTALL | re.IGNORECASE) is None:
                    if wordfilter['seconds']:
                        until = timestamp() + int(wordfilter['seconds'])
                    else:
                        until = 0

                    sql_query = "INSERT INTO `bans` (`ipstart`, `ipend`, `ipstr`, `boards`, `added`, `until`, `staff`, `reason`, `note`, `blind`) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
                    sql_params = (ip, ip, ip, wordfilter['boards'], timestamp(), until, "System", wordfilter['reason'], "Filter auto-ban", wordfilter['blind'])
                    InsertDb(sql_query, sql_params)
                    regenerateAccess()
                    raise UserError(wordfilter['reason'])
            elif wordfilter['action'] == 3:
                if not re.search(wordfilter['from'], message, re.DOTALL | re.IGNORECASE) is None:
                    raise UserError('<meta http-equiv="refresh" content="%s;url=%s" />%s' % (wordfilter['redirect_time'], wordfilter['redirect_url'], wordfilter['reason']))
    return message


def checkNamefilters(name, tripcode, ip, board):
    namefilters = FetchAll("SELECT * FROM `filters` WHERE `type` = '1'")

    for namefilter in namefilters:
        if namefilter["boards"]:
            boards = str2boards(namefilter["boards"])
        if not namefilter["boards"] or board in boards:
            # check if this filter applies
            match = False

            if namefilter['from'] and namefilter['from_trip']:
                # both name and trip filter
                if re.search(namefilter['from'], name, re.DOTALL | re.IGNORECASE) and tripcode == namefilter['from_trip']:
                    match = True
            elif namefilter['from'] and not namefilter['from_trip']:
                # name filter
                if re.search(namefilter['from'], name, re.DOTALL | re.IGNORECASE):
                    match = True
            elif not namefilter['from'] and namefilter['from_trip']:
                # trip filter
                if tripcode == namefilter['from_trip']:
                    match = True

        if match:
            # do action
            if namefilter['action'] == 0:
                raise UserError(namefilter['reason'])
            elif namefilter['action'] == 1:
                name = namefilter['to']
                tripcode = ''
                return name, tripcode
            elif namefilter['action'] == 2:
                # Ban
                if namefilter['seconds']:
                    until = timestamp() + int(namefilter['seconds'])
                else:
                    until = 0

                InsertDb("INSERT INTO `bans` (`ip`, `boards`, `added`, `until`, `staff`, `reason`, `note`, `blind`) VALUES (" +
                         "'" + _mysql.escape_string(ip) + "', '" + _mysql.escape_string(namefilter['boards']) +
                         "', " + str(timestamp()) + ", " + until + ", 'System', '" + _mysql.escape_string(namefilter['reason']) +
                         "', 'Name Auto-ban', '"+_mysql.escape_string(namefilter['blind'])+"')")
                regenerateAccess()
                raise UserError(namefilter['reason'])
            elif namefilter['action'] == 3:
                raise UserError('<meta http-equiv="refresh" content="%s;url=%s" />%s' % (namefilter['redirect_time'], namefilter['redirect_url'], namefilter['reason']))
    return name, tripcode