**
* 讀取文件具體操做
* @author Thinkpad
*
*/
public class ReadFile
{
private static SimpleDateFormat logTimeSDF = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); //日誌時間格式
private static String middle_file_path = ""; //中間文件路徑
private static String read_process_path = ""; //讀取進度文件路徑
//初始化配置文件
public static void initConfig() throws Exception{
InputStream in = null;
try{
File file = new File("config/config.properties"); //打成jar包時配置
in = new FileInputStream(file);
Properties prop = new Properties();
prop.load(in);
middle_file_path = prop.getProperty("middle_file_path");
read_process_path = prop.getProperty("read_process_path");
} catch(Exception e) {
throw e;
}finally{
if(in != null){
in.close();
}
}
}
public static String readMiddleFile() throws Exception
{
System.out.println("開始讀取【能見度中間文件】： " + logTimeSDF.format(System.currentTimeMillis()));
try{
if("".equals(middle_file_path) || "".equals(read_process_path)){
initConfig();
}
File fileDir = new File(middle_file_path); //中間文件目錄
if(fileDir.exists() && fileDir.isDirectory()){ //中間文件目錄是否存在
File[] fileList = fileDir.listFiles();
if(fileList.length > 0){ //有中間文件才啓動讀取流程
//第一步、讀取進度文件
BufferedReader in = null;
File mfp = new File(read_process_path);
if(!mfp.exists()){
mfp.mkdir();
}
Map<String, String> readProcessParams = new LinkedHashMap<>(); //文件讀取進度信息，保存讀取的進度文件信息
File processFile = new File(read_process_path + "/ReadMiddleFileLog.txt");
if(processFile.exists()){ //文件存在，讀取進度信息
try{
in = new BufferedReader(new InputStreamReader(new FileInputStream(processFile)));
String lineText = ""; //每行內容
while( (lineText = in.readLine()) != null ){ //讀到結束
lineText = lineText.trim();
String[] processInfo = lineText.split("#PARAMS#"); //以"#PARAMS#"分割
String fileName = processInfo[0];
readProcessParams.put(fileName, processInfo[1]);
}
}catch(Exception e){
throw e;
}finally {
if(in != null){
in.close(); in = null;
}
}
}else{
processFile.createNewFile(); //建立文件
}
//第二步、循環讀取目錄下中間結果文件
for(int i = 0, cc = fileList.length; i < cc; i ++){
File file = fileList[i];
String fileName = file.getName();
if(file.isFile() && fileName.indexOf(".txt") != -1){ //只讀取txt文件
try{
String processInfo = readProcessParams.get(fileName); //該文件讀取進度信息
int readLineNo = 0; //上次讀取行號，新文件從第一行開始讀
if(processInfo != null){ //該文件已經讀取
String[] params = processInfo.split("#PARAM#");
readLineNo = Integer.parseInt(params[0]); //上次讀取行號
}
List<String> insertSqlList = new ArrayList<>(); //入庫語句列表
StringBuilder insertSql = null; //入庫語句
int validCount = 0; //符合格式的新數據數量
in = new BufferedReader(new InputStreamReader(new FileInputStream(file)));
int lineNo = 0; //讀取第幾行
String lineText = ""; //每行內容
while( (lineText = in.readLine()) != null ){
lineNo ++;
if(lineNo > readLineNo){
lineText = lineText.trim();
//保存內容，在這入庫
String[] data = lineText.split(",");
if(data.length == 8){ //數量隨文件格式調整
validCount ++; //符合格式的數據+1
if(insertSql == null){ //500條記錄後重啓一個insert
insertSql =
new StringBuilder("insert into middle_file_data(file_name, col1, col2, col3, col4, col5, col6, col7, col8) values ");
}
//組織數據
insertSql.append("('" + fileName + "', '" + data[0] + "', '" + data[1] + "', '" + data[2] + "', '"
+ data[3] + "'," + " '" + data[4] + "', '" + data[5] + "', '" + data[6] + "', '" + data[7] + "'),");
//每500條記錄一個insert
if(validCount % 500 == 0){
insertSqlList.add(insertSql.substring(0, insertSql.length() -1)); //去除最後一個「,」
insertSql = null;
}
}
}
}
//第二步二、數據入庫
boolean insertFlag = true; //入庫成功標識
if(validCount > 0){
Connection conn = null;
Statement batchStat = null;
try{
if(insertSql != null) //處理最後500餘的數據
{
insertSqlList.add(insertSql.substring(0, insertSql.length()-1));
}
conn = CreateConnection.getConn();
conn.setAutoCommit(false); //不自動提交
batchStat = conn.createStatement();
for(String sql : insertSqlList)
{
batchStat.addBatch(sql);
}
batchStat.executeBatch(); //批量添加，保證事物性
conn.commit(); //提交
}catch(Exception e){
insertFlag = false; //設置入庫失敗
System.out.println(fileName + " 最新數據入庫失敗");
}finally {
CreateConnection.closeConn(null, batchStat, conn);
}
}/*else{
insertFlag 仍是true，沒有符合格式的新數據，讀取進度還要推動
}*/
if(lineNo > readLineNo && insertFlag){ //有讀取新記錄，而且入庫成功
String newProcessInfo = lineNo + "#PARAM#"; //新的讀取進度信息
readProcessParams.put(fileName, newProcessInfo); //放入進度集合，所有讀完後存入進度文件
}/*else{
集合中原記錄不變，和新進度一塊兒保存到進度文件中
}*/
}catch(Exception e){
throw e;
}finally {
if(in != null){in.close(); in = null;}
}
}
}
//第三步、更新進度文件
OutputStream out = null;
try{
out = new FileOutputStream(processFile);
StringBuilder precessInfoStr = new StringBuilder("");
//遍歷讀取信息集合，集合包含本次讀取修改過和未修改的文件讀取信息
for(Entry<String, String> entry : readProcessParams.entrySet()){
String fileName = entry.getKey();
String processInf0 = entry.getValue();
precessInfoStr.append(fileName + "#PARAMS#" + processInf0 + /*換行*/"\r\n");
}
out.write(precessInfoStr.toString().getBytes()); //寫入文件，覆蓋原來記錄
}catch(Exception e){
throw e;
}finally {
if(out != null){
out.close(); out = null;
}
}
}
}
}catch(Exception e){
throw e;
}
System.out.println("讀取【能見度中間文件】結束： " + logTimeSDF.format(System.currentTimeMillis()));
return "success";
}
}
創立數據庫鏈接
public class CreateConnection {
public static String dialect = "";
public static String driver = "";
public static String url = "";
public static String user = "";
public static String password = "";
public static Connection getConn() throws IOException {
Connection conn = null;
try {
if(driver == null || "".equals(driver))
{
readDBConfig();
}
Class.forName(driver);
conn = DriverManager.getConnection(url, user, password);// 獲取鏈接
} catch (ClassNotFoundException e) {
e.printStackTrace();
} catch (SQLException e) {
e.printStackTrace();
}
return conn;
}
public static void readDBConfig() throws IOException
{
InputStream in = null;
try{
File file = new File("config/config.properties");
in = new FileInputStream(file);
Properties prop = new Properties();
prop.load(in);
driver = prop.getProperty("jdbc.driverClassName");
url = prop.getProperty("jdbc.url");
user = prop.getProperty("jdbc.username");
password = prop.getProperty("jdbc.password");
} catch(Exception e) {
System.out.println("讀取數據庫配置文件失敗：");
e.printStackTrace();
} finally {
if(in != null){
in.close(); in = null;
}
}
}
public static void closeConn(ResultSet result, Statement statement, Connection conn)
{
try {
if(result != null && !result.isClosed()){
result.close();
}
} catch (SQLException e) {
e.printStackTrace();
}
try {
if(statement != null && !statement.isClosed()){
statement.close();
}
} catch (SQLException e) {
e.printStackTrace();
}
try {
if(conn != null && !conn.isClosed()){
conn.close();
}
} catch (SQLException e) {
e.printStackTrace();
}
}
}
/**
* 讀取線程
* @author Thinkpad
*
*/
public class ReadRunnable implements Runnable
{
public void run()
{
try{
ReadFile.readMiddleFile();
}catch(Exception e){
System.out.println("【能見度中間文件】讀取異常：" + e);
}catch(Error e){
System.out.println("【能見度中間文件】讀取錯誤：" + e);
}
}
}
/** * 讀取能見度中間結果文件入口 * @author Thinkpad * */public class MainRead { public static void main(String[] args) { ReadRunnable readRun = new ReadRunnable(); ScheduledExecutorService schedule = Executors.newSingleThreadScheduledExecutor(); schedule.scheduleWithFixedDelay(readRun, 5, 15, TimeUnit.SECONDS); //10秒後開始執行，每10分鐘執行一次 }}