933
社区成员




// 2.1 打开本地文件
QFile readFile(strFileFull);
if(!readFile.open((QIODevice::ReadOnly)))
{
qDebug("Failed to open file: %s!", strFileFull.toStdString().data());
m_pStdItemModel->item(n, 2)->setText(tr("打开本地文件失败"));
return false;
}
// 3. 向HDFS中写入文件
QString strWritePath = Global::GetConfig().GetHDFSMissionDataPath() + strFilePath; // if there has no '/' befor missionData, the path in HDFS will be /usr/<username>/temp/<strFileName>
// 3.1 上传前记录文件是否已存在
bool bExist = m_operateHDFS.exists(strWritePath.toStdString().data());
// 3.2 hdfs open file
if(!m_operateHDFS.open(strWritePath.toStdString().data(), O_WRONLY|O_CREAT))
{
m_pStdItemModel->item(n, 2)->setText(tr("HDFS文件打开失败"));
return false;
}
int nGB = 1024 * 1024 * 1024;
// 3.3 从本地文件读取nGB字节,写入HDFS,并设置文件偏移量
qDebug("写前偏移%ld", m_operateHDFS.tell());
for (int i=0; i<fileInfo.size()/nGB; i++)
{
// 3.3.1 从本地文件读取nGB字节,并设置文件偏移量
readFile.seek(i * nGB);
QByteArray qByteArry = readFile.read(nGB);
// 3.3.2 hdfs write
m_operateHDFS.write(qByteArry.data(), nGB);
qDebug("写中偏移%ld", m_operateHDFS.tell());
}
// 3.3.3 读取本地文件中剩下不足1G的数据,写入HDFS
int nRemaining = readFile.size() % nGB;
readFile.seek(fileInfo.size() / nGB * nGB);
QByteArray qByteArry = readFile.read(nRemaining);
if (-1 == m_operateHDFS.write(qByteArry.data(), nRemaining))
{
m_pStdItemModel->item(n, 2)->setText(tr("HDFS写入失败"));
return false;
}
qDebug("写后偏移%ld", m_operateHDFS.tell());
// 3.4 hdfs flush
if (-1 == m_operateHDFS.flush())
{
m_pStdItemModel->item(n, 2)->setText(tr("HDFS flush失败"));
return false;
}
// 3.4 hdfs close file
m_operateHDFS.close();
// 2.3 close file
readFile.close();
// 2.1 打开本地文件
QFile readFile(strFileFull);
if(!readFile.open((QIODevice::ReadOnly)))
{
qDebug("Failed to open file: %s!", strFileFull.toStdString().data());
m_pStdItemModel->item(n, 2)->setText(tr("打开本地文件失败"));
return false;
}
// 3. 向HDFS中写入文件
QString strWritePath = Global::GetConfig().GetHDFSMissionDataPath() + strFilePath; // if there is not '/' befor missionData, the path in HDFS will be /usr/<username>/temp/<strFileName>
// 3.1 上传前记录文件是否已存在
bool bExist = m_operateHDFS.exists(strWritePath.toStdString().data());
// 3.2 hdfs open file
if(!m_operateHDFS.open(strWritePath.toStdString().data(), O_WRONLY|O_CREAT))
{
m_pStdItemModel->item(n, 2)->setText(tr("HDFS文件打开失败"));
return false;
}
// 2.2 read file
int nGB = 1024 * 1024 * 1024;
// 3.3 hdfs write
qDebug("写前偏移%ld", m_operateHDFS.tell());
for (int i=0; i<fileInfo.size()/nGB; i++)
{
readFile.seek(i * nGB);
QByteArray qByteArry = readFile.read(nGB);
m_operateHDFS.write(qByteArry.data(), nGB);
qDebug("写中偏移%ld", m_operateHDFS.tell());
}
readFile.seek(fileInfo.size() / nGB * nGB);
if (-1 == m_operateHDFS.write(readFile.read(readFile.size() % nGB), readFile.size() % nGB))
{
m_pStdItemModel->item(n, 2)->setText(tr("HDFS写入失败"));
return false;
}
qDebug("写后偏移%ld", m_operateHDFS.tell());
// 3.4 hdfs flush
if (-1 == m_operateHDFS.flush())
{
m_pStdItemModel->item(n, 2)->setText(tr("HDFS flush失败"));
return false;
}
// 3.4 hdfs close file
m_operateHDFS.close();
// 2.3 close file
readFile.close();