安全日志系统数据网关实现

import json
import pyes
from django.shortcuts import render
from datetime import datetime
from elasticsearch import Elasticsearch
from django.contrib.auth.models import User, Group
from bone.models import Testcase
from rest_framework import viewsets
from xwing.serializers import UserSerializer, GroupSerializer,TestcaseSerializer
from django.http import HttpResponse
from django.views.decorators.csrf import csrf_exempt
from rest_framework.renderers import JSONRenderer
from rest_framework.parsers import JSONParser
from xwing.models import Snippet
from xwing.serializers import SnippetSerializer
es = Elasticsearch('127.0.0.1:9201')

class JSONResponse(HttpResponse):
    """
    An HttpResponse that renders it's content into JSON.
    """

    def __init__(self, data, **kwargs):
        content = JSONRenderer().render(data)
        kwargs['content_type'] = 'application/json'
        super(JSONResponse, self).__init__(content, **kwargs)

class UserViewSet(viewsets.ModelViewSet):
    queryset = User.objects.all()
    serializer_class = UserSerializer

class GroupViewSet(viewsets.ModelViewSet):
    queryset = Group.objects.all()
    serializer_class = GroupSerializer

class TestcaseViewSet(viewsets.ModelViewSet):
    queryset = Testcase.objects.all()
    serializer_class = TestcaseSerializer

@csrf_exempt
def get_phone_num(request, ver):
    if request.method == 'GET':
        return JSONResponse("GET_PHONE")
    elif request.method == 'POST':
        data = JSONParser().parse(request)

    flg_identity = data.has_key('identity')
    flg_token = data.has_key('token')
    flg_phone = data.has_key('phone')

    if not flg_token:    
        return JSONResponse('Token is empty!')

    if not data['token'] == 'testkey123123123':
        return JSONResponse('Token is invalid!')

    if not flg_phone:
        return JSONResponse('Phone is empty!')

    phone_num = data['phone']
    tmp_data = { 'query': { 'match': { 'phone': phone_num }}}
    res = es.search( index='data', body = tmp_data )
    ret_cnt = res['hits']['total']
    
    return JSONResponse(ret_cnt)

import sys
import json
import time
import logging
import optparse
import datetime
import string
from pygraylog.graylogapi import GraylogAPI
import base64
import requests

graylog_server_auth = {
    "url" : "http://127.0.0.1:12900",
    "username" : "graylog",
    "password" : "graylogpassword"
}

FIELDS_LIST = {
    "IPS" : ",src_ip,src_ip_geolocation,src_port,type,target_ip,target_ip_geolocation,target_port,time,username", #IPS Fileds    
}

STREAMS_LIST = {
    "IPS" : "abc09bfeb62567184383123", #IPS
}

def search_ips_stream(stream_name, info_map):
    api = GraylogAPI(graylog_server_auth['url'],
    graylog_server_auth['username'],
    graylog_server_auth['password'])
    filter_values = "streams:" + STREAMS_LIST[stream_name]
    fileds_values = FIELDS_LIST[stream_name]
    ret = api.search.universal.absolute.get(fields = fileds_values,
    query=info_map["query"],
    from_ = info_map["from"],
    to = info_map["to"],
    filter=filter_values,
    limit=info_map["limit"])
    return ret

def opts():
    usage = "usage: %prog -q ip -f fromdate -t todate -l limit -n username"
    parser = optparse.OptionParser(usage=usage)
    parser.add_option("-q", "--query", action="append", dest='query', nargs=1, type='string', help="query")
    parser.add_option("-f", "--fromdate", action="append", dest='fromdate', nargs=1, type='string', help="fromdate.")
    parser.add_option("-t", "--todate",action="append", dest='todate', nargs=1, type='string', help="todate.") 
    parser.add_option("-l", "--limit",action="append", dest='limit', nargs=1, type='string', help="limit.")
    parser.add_option("-n", "--filename", action="append", dest='filename', nargs=1, type='string', help="filename.")
    parser.add_option("--debug", action="store_const", dest='debug', help="enable debug log output", default=False, const=True)
    options, args = parser.parse_args()

    if len(args) < 1:
        parser.error('You need to give "subscribe" or "publish" as <action>.')

    if args[0] not in ['subscribe', 'publish', 'sendfile']:
        parser.error('You need to give "subscribe" or "publish" as <action>.')

    if options.debug:
        logging.basicConfig(level=logging.DEBUG)
    else:
        logging.basicConfig(level=logging.CRITICAL)

    action = args[0]
    data = None
    if action == 'publish':
        data = ' '.join(args[1:])
    elif action == 'sendfile':
        data = ' '.join(args[1:])
  
    return options, action, data

def main(opts, action, pubdata=None):
    query = options.query[0]
    fromdate = options.fromdate[0]
    todate = options.todate[0]
    limit = options.limit[0]
    filename = options.filename[0]
    info_map = {}
    info_map["type"] = "vpn"
    info_map["from"] = fromdate
    info_map["to"] = todate
    info_map["query"] = query
    info_map["limit"] = limit
    info_map["filename"] = filename

    ret = search_ips_stream("IPS",info_map)
    receive_data = json.loads(ret)
    ret_filename = './data/' + 'total' + '.csv'
    out_filename = './data/' + filename + '.csv'
    ret_object = open(ret_filename,'a')
    total = receive_data['total_results']
    if total > 0:
        uname = ""
        for item in receive_data['messages']:
            username = str(item['message']['username'])
            uname = uname + username + " "
            meta_data = "YES"+ "," + fromdate + "," + todate + "," +filename + "," + uname + "\n"
            ret_object.write(meta_data)
    else:
        meta_info = "NO"+ "," + fromdate + "," + todate + "," + filename + "\n"
        ret_object.write(meta_info)
        ret_object.close()
        output_object=open(out_filename,'w')
        
    for item in receive_data['messages']:
        username = str(item['message']['username'])
        estime=""
        timestamp = str(item['message']['timestamp'])
        meta_info = username + ',' + timestamp + ',' + '\n'
        output_object.write(meta_info)
        output_object.close()

if __name__ == '__main__':
    options, action, data = opts()
    try:
        sys.exit(main(options, action, pubdata=data))
    except KeyboardInterrupt:
        sys.exit(0)