81,094
社区成员
发帖
与我相关
我的任务
分享
public void saveLogFiles() throws IOException, SQLException, ClassNotFoundException {
Long count = 0L; // 重复记录条数
BufferedReader br = null;
String syslogPath = "D:\\syslog";
File syslog = new File(syslogPath);
Connection con = backUpDao.getConnection();
List<LogInfo> logInfos = new ArrayList<LogInfo>();
for (String dayFileName : syslog.list()) {
String daFilePath = syslogPath + "\\" + dayFileName;
File dayFile = new File(daFilePath);
if (dayFile.isFile()) {
// 如果在syslog文件夹下的当前文件不是日志文件夹,则判断下一个文件
continue;
}
for (String logFileName : dayFile.list()) {
String logFilePath = syslogPath + "\\" + dayFileName + "\\" + logFileName;
System.out.println("正在分析日志文件:" + logFilePath);
File logFile = new File(logFilePath);
try {
br = new BufferedReader(new FileReader(logFile));
String line = "";
while ((line = br.readLine()) != null) {
int indexOfLen = line.indexOf("len");
int indexOfFrom = line.indexOf("from");
if (indexOfLen == -1 || indexOfFrom == -1) {
// 如果找不到对应字符,则跳过该行
continue;
}
String srcIpAndPort = getSrcIpFromLog(line);
String srcIp = srcIpAndPort.split(":")[0];
String destIpAndPort = getDesIpFromLog(line);
String destIp = destIpAndPort.split(":")[0];
String date = getDateFromLog(line, indexOfLen, indexOfFrom);
Calendar cal = Calendar.getInstance();
long startDate = sdf.parse(date).getTime();
cal.setTimeInMillis(startDate);
cal.set(Calendar.DATE, cal.get(Calendar.DATE) +1);
long endDate = cal.getTimeInMillis();
String mac = getMacFromLog(line);
// 该条记录已存在相同IP,不在写入数据库
if (backUpDao.getByIp(con, srcIp, destIp, startDate, endDate, LogInfo.Area.XIAOSHAN.toString())) {
System.out.println("repeat log count: " + ++count + "; the content is " + line);
continue;
}
// 封装LogInfo
LogInfo logInfo = new LogInfo();
logInfo.setMac(mac);
logInfo.setSrcIp(srcIpAndPort);
logInfo.setDesIp(destIpAndPort);
logInfo.setDate(date);
logInfo.setContent(line);
logInfo.setArea(LogInfo.Area.XIAOSHAN);
logInfos.add(logInfo);
}
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} catch (ParseException e) {
e.printStackTrace();
} finally {
br.close();
backUpDao.closeConnection(null, null, con);
}
}
}
System.out.println("分析完成...\n开始写入数据库...");
backUpDao.saveLogoInfo(logInfos);
}
public boolean getByIp(Connection con, String srcIp, String destIp, long startTimestamp, long endTimestamp, String area) throws SQLException {
PreparedStatement ps = null;
ResultSet rs = null;
try {
if (con == null) {
con = getConnection();
}
// 查询该条记录中的源IP和目标IP在数据库中当天记录里面是否存在
String sql = "select * from log l where l.src_ip_and_port like ? and l.dest_ip_and_port like ? and unix_timestamp(l.date) >= ? and unix_timestamp(l.date) < ? and l.area = ?";
ps = con.prepareStatement(sql);
ps.setString(1, srcIp + "%");
ps.setString(2, destIp + "%");
// mysqk的unix_timestamp函数以秒为准,所以下面要除以1000
ps.setLong(3, startTimestamp / 1000);
ps.setLong(4, endTimestamp / 1000);
ps.setString(5, area);
rs = ps.executeQuery();
while (rs.next()) {
return true;
}
} catch (Exception e) {
e.printStackTrace();
} finally {
closeConnection(rs, ps, null);
}
return false;
}
// 该条记录已存在相同IP,不在写入数据库
if (backUpDao.getByIp(con, srcIp, destIp, startDate, endDate, LogInfo.Area.XIAOSHAN.toString())) {
System.out.println("repeat log count: " + ++count + "; the content is " + line);
continue;
}
这里,我先把所有数据都分析完了再插入数据库。[/quote]
getConnection()
检查一下是不是每次getConnection 都新开了一个连接
closeConnection(rs, ps, null);
这个方法好像并没有关闭Connection 连接
如果直是这样,有可能你打开的连接数超过了数据库的最大连接。
解决方法有:
1. getConnection 的时候重复利用已经打开的连接 (比较靠谱)
2. closeConnection 的时候关闭连接 (有点靠谱)
3. 把数据库连接数设置大一点 (不是太靠谱,因为你的程序设计的不好,再大的连接数都会爆的一刻)[/quote]
以上观点当说没过,没仔细看你的程序,应该不存在过多连接的问题 // 该条记录已存在相同IP,不在写入数据库
if (backUpDao.getByIp(con, srcIp, destIp, startDate, endDate, LogInfo.Area.XIAOSHAN.toString())) {
System.out.println("repeat log count: " + ++count + "; the content is " + line);
continue;
}
这里,我先把所有数据都分析完了再插入数据库。[/quote]
getConnection()
检查一下是不是每次getConnection 都新开了一个连接
closeConnection(rs, ps, null);
这个方法好像并没有关闭Connection 连接
如果直是这样,有可能你打开的连接数超过了数据库的最大连接。
解决方法有:
1. getConnection 的时候重复利用已经打开的连接 (比较靠谱)
2. closeConnection 的时候关闭连接 (有点靠谱)
3. 把数据库连接数设置大一点 (不是太靠谱,因为你的程序设计的不好,再大的连接数都会爆的一刻)开始备份日志文件...
正在分析日志文件:D:\syslog\20131123\0015.log
com.mysql.jdbc.CommunicationsException: The driver was unable to create a connection due to an inability to establish the client portion of a socket.
This is usually caused by a limit on the number of sockets imposed by the operating system. This limit is usually configurable.
For Unix-based platforms, see the manual page for the 'ulimit' command. Kernel or system reconfiguration may also be required.
For Windows-based platforms, see Microsoft Knowledge Base Article 196271 (Q196271).
at com.mysql.jdbc.Connection.createNewIO(Connection.java:2847)
at com.mysql.jdbc.Connection.<init>(Connection.java:1555)
at com.mysql.jdbc.NonRegisteringDriver.connect(NonRegisteringDriver.java:285)
at java.sql.DriverManager.getConnection(DriverManager.java:582)
at java.sql.DriverManager.getConnection(DriverManager.java:185)
at com.hjc.backup.dao.BackUpDao.getConnection(BackUpDao.java:30)
at com.hjc.backup.dao.BackUpDao.getByIp(BackUpDao.java:109)
at com.hjc.backup.service.LogService.saveLogFiles(LogService.java:91)
at com.hjc.backup.service.LogService.main(LogService.java:280)
频繁创建connection,到1000个就报Java heap space错误,目前还没在tomcat上面跑,就在控制台跑跑的。[/quote]Java heap space 是 java 报的,还是 db 报的?
你不会 java 和 db 跑在一台了?[/quote] // 该条记录已存在相同IP,不在写入数据库
if (backUpDao.getByIp(con, srcIp, destIp, startDate, endDate, LogInfo.Area.XIAOSHAN.toString())) {
System.out.println("repeat log count: " + ++count + "; the content is " + line);
continue;
}
这里,我先把所有数据都分析完了再插入数据库。