web_stats Namespace Reference

Functions
def	collect_and_log ()
def	__create_es_connection__ ()
def	search_es (es_conn)
def	extract_messages_from_results (results)
def	map_messages_into_bins (messages)
def	process_access_log_bins (bins)
def	write_access_log_file (filename, lines)

Variables
	LOGGER = logging.getLogger(__name__)
	ELASTICSEARCH_URL = os.environ['ELASTICSEARCH_URL']
	LOG_OUTPUT_DIR = os.environ.get('LOG_OUTPUT_DIR')
	UID = pwd.getpwnam('www-data').pw_uid

Detailed Description

Collect web stats from elasticsearch for later parsing

Function Documentation

collect_and_log()

def web_stats.collect_and_log

(

)

Collect web stats and write to log

Definition at line 22 of file web_stats.py.

def collect_and_log():
    '''Collect web stats and write to log'''
    es_conn = __create_es_connection__()
    access_log = os.path.join(LOG_OUTPUT_DIR, 'access_log')
    assert es_conn.ping()
    compose(partial(write_access_log_file, access_log),
            extract_messages_from_results,
            search_es)(es_conn)

extract_messages_from_results()

def web_stats.extract_messages_from_results

(

results

)

pull messages from es search results

   Returns a generator for results.

Definition at line 61 of file web_stats.py.

def extract_messages_from_results(results):
    '''pull messages from es search results

       Returns a generator for results.
    '''
    return (hit['_source']['message'] for hit in results['hits']['hits'])

map_messages_into_bins()

def web_stats.map_messages_into_bins

(

messages

)

map log items (messages) into "bins"

Definition at line 69 of file web_stats.py.

def map_messages_into_bins(messages):
    '''map log items (messages) into "bins"'''
    bins = {}
    for message in messages:
        # Probably an improper split...
        key = message.split('-')[1].strip() or "default"
        if key in bins:
            bins[key].extend([message])
        else:
            bins[key] = [message]
    return bins

process_access_log_bins()

def web_stats.process_access_log_bins

(

bins

)

process bins and write access logs

Definition at line 82 of file web_stats.py.

def process_access_log_bins(bins):
    '''process bins and write access logs'''
    if not os.path.exists(LOG_OUTPUT_DIR):
        os.mkdir(LOG_OUTPUT_DIR)
    for key in bins.keys():
        filename = os.path.join(LOG_OUTPUT_DIR, '%s_access.log' % key)
        write_access_log_file(filename, bins[key])

search_es()

def web_stats.search_es

(

es_conn

)

perform es search

Definition at line 38 of file web_stats.py.

def search_es(es_conn):
    '''perform es search'''
    assert es_conn
    now = datetime.now()
    yesterday = now - timedelta(days=1)
    day_before_yesterday = yesterday - timedelta(days=1)
    query_body = {
        "query": {
            "bool": {
                "filter": [
                    {"match": {"image_tag": "odyssey_web"}},
                    {"range": {"timestamp": {
                        # pylint: disable=no-member
                        "gte": day_before_yesterday.isoformat(),
                        "lte": yesterday.isoformat()
                    }}},
                ]
            }
        }
    }
    return es_conn.search(body=query_body)

write_access_log_file()

def web_stats.write_access_log_file	(		filename,
			lines
	)

Write out access log

Definition at line 91 of file web_stats.py.

def write_access_log_file(filename, lines):
    '''Write out access log'''
    with open(filename, 'w') as access_log:
        access_log.write('\n'.join(lines))
    return filename