#! /usr/bin/env python  

# -*- coding: utf-8 -*-  

import os  

import getopt  

import sys  

import fnmatch    

from datetime import datetime,timedelta,date  

#判断是否为新一天的开始,数据文件是按日保存,凌晨定时任务需要对前日的数据进行增量计算  

def is_newday_begin(app,daystr):  

    file_day_file='/var/local/diff_file_storage/day_'+app  

    if os.path.exists(file_day_file):  

        try:  

            day_file=open(file_day_file,'r')  

            oldday=day_file.read()  

            if oldday != daystr:  

                return True  

        finally:  

            day_file.close()  

    return False  

def read_position(app,daystr,file):  

    file_day_file='/var/local/diff_file_storage/day_'+app  

    if os.path.exists(file_day_file):  

        try:  

            day_file=open(file_day_file,'r')  

            oldday=day_file.read()  

            if oldday != daystr:  

                day_file.close()  

                day_file=open(file_day_file,'w+')  

                day_file.write(daystr)  

                write_position(daystr,file,0)  

        finally:  

            day_file.close()  

    else:  

        try:  

            day_file=open(file_day_file,'w+')  

            day_file.write(daystr)  

        finally:  

            day_file.close()   

    position=None  

    open_file=None  

    position_file=file+'_'+daystr  

    try:  

        if os.path.exists(position_file):  

            open_file=open(position_file,'r')  

            position = long(open_file.read())  

        else:  

            position=0  

    finally:  

        if open_file is not None:  

            open_file.close()  

    print 'read from ',position_file  

    return position  

def write_position(daystr,file,position):  

    open_file=None  

    position_file=file+'_'+daystr  

    try:  

        open_file=open(position_file,'w+')  

        open_file.write(str(position))  

    finally:  

        if open_file is not None:  

            open_file.close()  

    print 'write to ',position_file  

def read_diff(app,day,outpath,times):  

    filepath='/data/dc_files/'+app+day.strftime('/%Y/%m')  

    if is_newday_begin(app,day.strftime('%Y-%m-%d')) and times <= 0:  

        read_diff(app,day+timedelta(days=-1),outpath,times+1)  

        print 'read_diff old day : ',day  

    #get old position of file  

    file_position=0  

    file_position_dir='/var/local/diff_file_storage/position/'  

    filenames=os.listdir(filepath)  

    if not filepath.endswith('/'):  

        filepath = filepath + '/'  

    filepre = app+'_'+day.strftime('%Y-%m-%d')+'_'   

        target_file=open(outpath,'a')  

    try:  

        for line in filenames:  

            if line.startswith(filepre):  

                print 'read_diff: flush file ',line  

                file_position_path=file_position_dir+app+'_'+line.replace(filepre,'')  

                #read position from temp  

                oldposition = read_position(app,day.strftime('%Y-%m-%d'),file_position_path)  

                position=oldposition  

                print 'read_diff: old position is ',position  

                #read data,change position  

                try:  

                    source_file=open(filepath+line,'r')  

                    #target_file=open(outpath,'w')  

                    source_file.seek(oldposition)  

                    for temp_line in source_file:  

                        target_file.write(temp_line)  

                        position = position+len(temp_line)  

                finally:  

                    source_file.close()  

                    if position==oldposition:  

                        #os.remove(outpath)  

                        print outpath,' empty data'   

                #write position back to temp  

                write_position(day.strftime('%Y-%m-%d'),file_position_path,position)  

                print 'read_diff: new position is ',position  

    finally:  

        target_file.close()