diff options
| -rw-r--r-- | cgi/framework.py | 43 | ||||
| -rw-r--r-- | cgi/img.py | 133 | ||||
| -rw-r--r-- | cgi/manage.py | 11 | ||||
| -rw-r--r-- | cgi/post.py | 9 | ||||
| -rwxr-xr-x | cgi/weabot.py | 21 |
5 files changed, 97 insertions, 120 deletions
diff --git a/cgi/framework.py b/cgi/framework.py index 5f95303..8c39f81 100644 --- a/cgi/framework.py +++ b/cgi/framework.py @@ -9,6 +9,7 @@ import socket import _mysql import urllib import re +import logging from Cookie import SimpleCookie from settings import Settings @@ -22,7 +23,7 @@ def setBoard(dir): """ if not dir: raise UserError, _("The specified board is invalid.") - logTime("Seteando el board " + dir) + logging.debug("Seteando el board " + dir) board = FetchOne( "SELECT * FROM `boards` WHERE `dir` = '%s' LIMIT 1" % _mysql.escape_string(dir)) if not board: @@ -32,7 +33,6 @@ def setBoard(dir): "SELECT * FROM `boards_filetypes` INNER JOIN `filetypes` ON filetypes.id = boards_filetypes.filetypeid WHERE `boardid` = %s ORDER BY `ext` ASC" % _mysql.escape_string(board['id'])) board["filetypes_ext"] = [filetype['ext'] for filetype in board['filetypes']] - logTime("Board seteado.") Settings._.BOARD = board @@ -474,42 +474,3 @@ def send_mail(subject, srcmsg): s.sendmail(me, [you], msg.as_string()) s.quit() - -class weabotLogger: - def __init__(self): - self.times = [] - - def log(self, message): - self.times.append([time.time(), message]) - - def allTimes(self): - output = "Time Logged action\n--------------------------\n" - start = self.times[0][0] - for time in self.times: - difference = str(time[0] - start) - difference_split = difference.split(".") - if len(difference_split[0]) < 2: - difference_split[0] = "0" + difference_split[0] - - if len(difference_split[1]) < 7: - difference_split[1] = ( - "0" * (7 - len(difference_split[1]))) + difference_split[1] - elif len(difference_split[1]) > 7: - difference_split[1] = difference_split[1][:7] - - output += ".".join(difference_split) + " " + time[1] + "\n" - - return output - - -logger = weabotLogger() - - -def logTime(message): - global logger - logger.log(message) - - -def logTimes(): - global logger - return logger.allTimes() @@ -4,6 +4,7 @@ import math #import random import os import subprocess +import logging from StringIO import StringIO from settings import Settings @@ -30,6 +31,7 @@ def processImage(post, data, t, originalname, spoiler=False): # get image information content_type, width, height, size, extra = getImageInfo(data) + logging.info("{} {} {}".format(content_type, width, height)) # check the size is fine if size > int(board["maxsize"])*1024: @@ -87,66 +89,63 @@ def processImage(post, data, t, originalname, spoiler=False): # Do we need to thumbnail it? if not used_filetype['image']: # make thumbnail + logging.debug("Generating thumbnail") file_thumb_width, file_thumb_height = getThumbDimensions( width, height, maxsize) - - if used_filetype['ffmpeg_thumb'] == '1': - # use ffmpeg to make thumbnail - logTime("Generating thumbnail") - - if used_filetype['mime'][:5] == 'video': - retcode = subprocess.call([ - Settings.FFMPEG_PATH, '-strict', '-2', '-ss', '0', '-i', file_path, - '-v', 'quiet', '-an', '-vframes', '1', '-f', 'mjpeg', '-vf', 'scale=%d:%d' % ( - file_thumb_width, file_thumb_height), - '-threads', '1', file_thumb_path]) - if spoiler: - args = [Settings.CONVERT_PATH, file_thumb_path, "-limit", "thread", "1", "-background", "white", "-flatten", "-resize", "%dx%d" % (file_thumb_width, file_thumb_height), "-blur", "0x12", "-gravity", "center", "-fill", "rgba(0,0,0, .6)", "-draw", "rectangle 0,%d,%d,%d" % ( - (file_thumb_height/2)-10, file_thumb_width, (file_thumb_height/2)+7), "-fill", "white", "-annotate", "0", "Alerta de spoiler", "-quality", str(Settings.THUMB_QUALITY), file_thumb_path] - retcode = subprocess.call(args) - elif used_filetype['mime'][:5] == 'audio': - # we do an exception and use png for audio waveform thumbnails since they - # 1. are smaller 2. allow for transparency - file_thumb_name = file_thumb_name[:-3] + "png" - file_thumb_path = file_thumb_path[:-3] + "png" - file_mobile_path = file_mobile_path[:-3] + "png" - file_cat_path = file_cat_path[:-3] + "png" - - if int(board['thumb_px']) > 149: - file_thumb_width = board['thumb_px'] - file_thumb_height = float(int(board['thumb_px'])/2) - else: - file_thumb_width = 150 - file_thumb_height = 75 - - retcode = subprocess.call([ - Settings.FFMPEG_PATH, '-t', '300', '-i', file_path, - '-filter_complex', 'showwavespic=s=%dx%d:split_channels=1' % ( + + try: + if used_filetype['ffmpeg_thumb'] == '1': + # use ffmpeg to make thumbnail + if used_filetype['mime'][:5] == 'video': + # Create preview for video AND spoiler it if necessary + call_wrap([ + Settings.FFMPEG_PATH, '-strict', '-2', '-ss', '0', '-i', file_path, + '-v', 'quiet', '-an', '-vframes', '1', '-f', 'mjpeg', '-vf', 'scale=%d:%d' % ( + file_thumb_width, file_thumb_height), + '-threads', '1', file_thumb_path]) + if spoiler: + call_wrap([Settings.CONVERT_PATH, file_thumb_path, "-limit", "thread", + "1", "-background", "white", "-flatten", "-resize", + "%dx%d" % (file_thumb_width, file_thumb_height), "-blur", "0x12", + "-gravity", "center", "-fill", "rgba(0,0,0, .6)", + "-draw", "rectangle 0,%d,%d,%d" % ((file_thumb_height/2)-10, file_thumb_width, (file_thumb_height/2)+7), + "-fill", "white", "-font", "Liberation-Sans", "-annotate", "0", "Alerta de spoiler", + "-quality", str(Settings.THUMB_QUALITY), file_thumb_path]) + elif used_filetype['mime'][:5] == 'audio': + # we do an exception and use png for audio waveform thumbnails since they + # 1. are smaller 2. allow for transparency + file_thumb_name = file_thumb_name[:-3] + "png" + file_thumb_path = file_thumb_path[:-3] + "png" + file_mobile_path = file_mobile_path[:-3] + "png" + file_cat_path = file_cat_path[:-3] + "png" + + if int(board['thumb_px']) > 149: + file_thumb_width = board['thumb_px'] + file_thumb_height = float(int(board['thumb_px'])/2) + else: + file_thumb_width = 150 + file_thumb_height = 75 + + call_wrap([Settings.FFMPEG_PATH, '-t', '300', '-i', file_path, + '-filter_complex', 'showwavespic=s=%dx%d:split_channels=1' % ( int(file_thumb_width), int(file_thumb_height)), - '-frames:v', '1', '-threads', '1', file_thumb_path]) -# elif used_filetype['mime'] == 'application/x-shockwave-flash' or used_filetype['mime'] == 'mime/x-shockwave-flash': -# retcode = subprocess.call([ -# './ffmpeg', '-i', file_path, '-vcodec', 'mjpeg', '-vframes', '1', '-an', '-f', 'rawvideo', -# '-vf', 'scale=%d:%d' % (file_thumb_width, file_thumb_height), '-threads', '1', file_thumb_path]) - - if retcode != 0: - os.remove(file_path) - raise UserError, _("Thumbnail creation failure.") + ' ('+str(retcode)+')' - else: - # use imagemagick to make thumbnail - args = [Settings.CONVERT_PATH, file_path, "-limit", "thread", "1", "-background", - "white", "-flatten", "-resize", "%dx%d" % (file_thumb_width, file_thumb_height)] - if spoiler: - args += ["-blur", "0x12", "-gravity", "center", "-fill", "rgba(0,0,0, .6)", "-draw", "rectangle 0,%d,%d,%d" % ( - (file_thumb_height/2)-10, file_thumb_width, (file_thumb_height/2)+7), "-fill", "white", "-annotate", "0", "Alerta de spoiler"] - args += ["-quality", str(Settings.THUMB_QUALITY), file_thumb_path] - - # generate thumbnails - logTime("Generating thumbnail") - retcode = subprocess.call(args) - if retcode != 0: - os.remove(file_path) - raise UserError, _("Thumbnail creation failure.") + ' ('+str(retcode)+')' + '-frames:v', '1', '-threads', '1', file_thumb_path]) + else: + # use imagemagick to make thumbnail + args = [Settings.CONVERT_PATH, file_path, "-limit", "thread", "1", "-background", + "white", "-flatten", "-resize", "%dx%d" % (file_thumb_width, file_thumb_height)] + if spoiler: + args += ["-blur", "0x12", "-gravity", "center", "-fill", "rgba(0,0,0, .6)", "-draw", "rectangle 0,%d,%d,%d" % ( + (file_thumb_height/2)-10, file_thumb_width, (file_thumb_height/2)+7), "-fill", "white", + "-font", "Liberation-Sans", "-annotate", "0", "Alerta de spoiler"] + args += ["-quality", str(Settings.THUMB_QUALITY), file_thumb_path] + + # generate thumbnails + call_wrap(args) + except subprocess.CalledProcessError, e: + os.remove(file_path) + logging.error("Thumbnail creation failure: " + e.output) + raise UserError, _("Thumbnail creation failure.") + ' ('+str(e.returncode)+')' # check if thumbnail was truly created try: @@ -293,6 +292,22 @@ def getImageInfo(data): except ValueError: pass + # handle WebP + if data[:4] == b'RIFF' and data[8:12] == b'WEBP': + chunk = data[12:] + if chunk[:4] == b"VP8 " and chunk[11:14] == b"\x9d\x01\x2a": + # Lossy VP8 + w, h = struct.unpack("HH", chunk[14:18]) + width = w & 0x3fff + height = h & 0x3fff + content_type = "image/webp" + elif chunk[:4] == b"VP8L": + # Lossless VP8 + b0, b1, b2, b3 = struct.unpack("BBBB", chunk[9:13]) + width = 1 + (((b1 & 0x3F) << 8) | b0) + height = 1 + (((b3 & 0xF) << 10) | (b2 << 2) | ((b1 & 0xC0) >> 6)) + content_type = "image/webp" + # handle WebM elif (size >= 4) and data.startswith("\x1A\x45\xDF\xA3"): content_type = "video/webm" @@ -371,6 +386,10 @@ def ffprobe_f(filename): return json.loads(out) +def call_wrap(args): + subprocess.check_output(args, stderr=subprocess.STDOUT) + + def getThumbDimensions(width, height, maxsize): """ Calculate dimensions to use for a thumbnail with maximum width/height of diff --git a/cgi/manage.py b/cgi/manage.py index f21e7e8..5def5c5 100644 --- a/cgi/manage.py +++ b/cgi/manage.py @@ -3,9 +3,8 @@ import _mysql import os import cgi import shutil -import imaplib -import poplib import datetime +import logging from database import * from settings import Settings @@ -40,8 +39,8 @@ def manage(self, path_split): str(timestamp() - 604800)) # one week else: page += _('Incorrect username/password.') - logAction('', 'Failed log-in. U:'+_mysql.escape_string( - self.formdata['username'])+' IP:'+self.environ["REMOTE_ADDR"]) + logAction('', 'Failed log-in. U:'+_mysql.escape_string(self.formdata['username'])+' IP logged.') + logging.warn("Failed log-in. U:{} IP:{}".format(self.formdata['username'], self.environ["REMOTE_ADDR"])) else: # Validate existing session manage_cookie = getCookie(self, 'weabot_manage') @@ -1117,7 +1116,7 @@ def manage(self, path_split): # Table if 'board' in self.formdata.keys() and self.formdata['board'] != 'all': cboard = self.formdata['board'] - posts = FetchAll("SELECT posts.id, posts.timestamp, timestamp_formatted, IS_DELETED, INET6_NTOA(posts.ip) as ip, posts.message, dir, boardid FROM `posts` INNER JOIN `boards` ON boardid = boards.id WHERE `dir` = '%s' AND IS_DELETED %s ORDER BY `timestamp` DESC LIMIT %d, %d" % ( + posts = FetchAll("SELECT posts.id, posts.timestamp, timestamp_formatted, IS_DELETED, INET6_NTOA(posts.ip) AS ip, posts.message, dir, boardid FROM `posts` INNER JOIN `boards` ON boardid = boards.id WHERE `dir` = '%s' AND IS_DELETED %s ORDER BY `timestamp` DESC LIMIT %d, %d" % ( _mysql.escape_string(self.formdata['board']), _mysql.escape_string(type_condition), currentpage*pagesize, pagesize)) try: totals = FetchOne("SELECT COUNT(id) FROM `posts` WHERE IS_DELETED %s AND `boardid` = %s" % ( @@ -1126,7 +1125,7 @@ def manage(self, path_split): skip = True else: cboard = 'all' - posts = FetchAll("SELECT posts.id, posts.timestamp, timestamp_formatted, IS_DELETED, posts.ip, posts.message, dir FROM `posts` INNER JOIN `boards` ON boardid = boards.id WHERE IS_DELETED %s ORDER BY `timestamp` DESC LIMIT %d, %d" % ( + posts = FetchAll("SELECT posts.id, posts.timestamp, timestamp_formatted, IS_DELETED, INET6_NTOA(posts.ip) AS ip, posts.message, dir FROM `posts` INNER JOIN `boards` ON boardid = boards.id WHERE IS_DELETED %s ORDER BY `timestamp` DESC LIMIT %d, %d" % ( _mysql.escape_string(type_condition), currentpage*pagesize, pagesize)) totals = FetchOne("SELECT COUNT(id) FROM `posts` WHERE IS_DELETED %s" % _mysql.escape_string(type_condition), 0) diff --git a/cgi/post.py b/cgi/post.py index da2ad47..e27b971 100644 --- a/cgi/post.py +++ b/cgi/post.py @@ -7,6 +7,7 @@ import threading import Queue import _mysql import formatting +import logging from database import * from template import * @@ -49,7 +50,7 @@ class Post(object): return self.post def insert(self): - logTime("Insertando Post") + logging.info("Insertando Post") post_values = [] for key, value in self.post.iteritems(): @@ -715,7 +716,7 @@ def deletePost(postid, password, deltype='0', imageonly=False, quick=False): if int(post["parentid"]) == 0: deleteReplies(post) - logTime("Deleting post " + str(postid)) + logging.info("Deleting post " + str(postid)) if deltype != '0' and post["parentid"] != '0': # Soft delete (recycle bin) UpdateDb("UPDATE `posts` SET `IS_DELETED` = %s WHERE `boardid` = %s AND `id` = %s LIMIT 1" % (deltype, board["id"], post["id"])) @@ -788,7 +789,7 @@ def trimThreads(): """ Delete any threads which have passed the MAX_THREADS setting """ - logTime("Trimming threads") + logging.debug("Trimming threads") board = Settings._.BOARD archived = False @@ -1026,7 +1027,7 @@ def regenerateHome(): """ Update index.html in the boards directory with useful data for users """ - logTime("Updating home") + logging.debug("Updating home") t = datetime.datetime.now() limit = Settings.HOME_LASTPOSTS diff --git a/cgi/weabot.py b/cgi/weabot.py index 3c5d0f5..f92e777 100755 --- a/cgi/weabot.py +++ b/cgi/weabot.py @@ -8,6 +8,7 @@ import time import datetime import random import cgi +import logging import _mysql from Cookie import SimpleCookie @@ -33,6 +34,9 @@ _LOG = False class weabot(object): def __init__(self, environ, start_response): global _DEBUG + + logging.basicConfig(filename='weabot.log', format='%(asctime)s %(levelname)s %(message)s', level=logging.INFO) + self.environ = environ if self.environ["PATH_INFO"].startswith("/weabot.py/"): self.environ["PATH_INFO"] = self.environ["PATH_INFO"][11:] @@ -48,7 +52,6 @@ class weabot(object): 'weabot', './locale', languages=[Settings.LANG]) lang.install() - logTime("**Start**") if _DEBUG: import cProfile @@ -61,6 +64,8 @@ class weabot(object): except UserError, message: self.error(message) except Exception, inst: + logging.exception(inst) + import sys import traceback exc_type, exc_value, exc_traceback = sys.exc_info() @@ -70,12 +75,6 @@ class weabot(object): # close database and finish CloseDb() - logTime("**End**") - - if _LOG: - logfile = open(Settings.ROOT_DIR + "weabot.txt", "w") - logfile.write(logTimes()) - logfile.close() def __iter__(self): self.handleResponse() @@ -732,8 +731,6 @@ class weabot(object): if board["slip"] == '2': if hide_end: host = '★' - elif addressIsTor(ip): - host = 'onion' else: host = getHost(ip) if host: @@ -767,7 +764,7 @@ class weabot(object): if hide_end or addressIsTor(ip): country = '??' else: - country = getCountry(ip) + country = getCountry(ip) or '??' post["name"] += " <em>[%s]</em>" % country # set expiration date if necessary @@ -812,7 +809,7 @@ class weabot(object): post["message"]), _mysql.escape_string(board["id"]), _mysql.escape_string(str(postid)))) # do operations if replying to a thread (bump, autoclose, update cache) - logTime("Updating thread") + logging.debug("Updating thread") thread_length = None if post["parentid"]: # get length of the thread @@ -1086,7 +1083,7 @@ if __name__ == "__main__": # Psyco is not required, however it will be used if available try: import psyco - logTime("Psyco se ha instalado") + logging.debug("Psyco se ha instalado") psyco.bind(tenjin.helpers.to_str) psyco.bind(weabot.run, 2) psyco.bind(getFormData) |