// -std=c++17 -O3 -march=native -pthread -Wall -Wextra
#include <iostream>
#include <fstream>
#include <vector>
#include <string>
#include <cstring>
#include <thread>
#include <mutex>
#include <algorithm>
#include <cmath>
#include <atomic>
#include <cstdint>
#include <filesystem>
#include <sstream>
#include <iomanip>
#include <functional>namespace fs = std::filesystem;// Data type enum
enum class DataType {UINT4,UINT8,UINT12,UINT16,UINT32,UINT64,INT8,INT16,INT32,INT64,FP16,FLOAT32,FLOAT64,UNKNOWN
};// FP16 to float conversion
float fp16ToFloat(uint16_t h) {uint32_t sign = (h >> 15) & 0x1;uint32_t exp = (h >> 10) & 0x1F;uint32_t mant = h & 0x3FF;uint32_t f;if (exp == 0) {if (mant == 0) {f = sign << 31;} else {// Subnormal numberexp = 1;while ((mant & 0x400) == 0) {mant <<= 1;exp--;}mant &= 0x3FF;f = (sign << 31) | ((exp + 112) << 23) | (mant << 13);}} else if (exp == 31) {// Infinity or NaNf = (sign << 31) | (0xFF << 23) | (mant << 13);} else {// Normal numberf = (sign << 31) | ((exp + 112) << 23) | (mant << 13);}return *reinterpret_cast<float*>(&f);
}std::string dtypeToString(DataType dt) {switch (dt) {case DataType::UINT4: return "uint4";case DataType::UINT8: return "uint8";case DataType::UINT12: return "uint12";case DataType::UINT16: return "uint16";case DataType::UINT32: return "uint32";case DataType::UINT64: return "uint64";case DataType::INT8: return "int8";case DataType::INT16: return "int16";case DataType::INT32: return "int32";case DataType::INT64: return "int64";case DataType::FP16: return "fp16";case DataType::FLOAT32: return "float32";case DataType::FLOAT64: return "float64";default: return "unknown";}
}DataType parseDataType(const std::string& s) {if (s == "uint4") return DataType::UINT4;if (s == "uint8") return DataType::UINT8;if (s == "uint12") return DataType::UINT12;if (s == "uint16") return DataType::UINT16;if (s == "uint32") return DataType::UINT32;if (s == "uint64") return DataType::UINT64;if (s == "int8") return DataType::INT8;if (s == "int16") return DataType::INT16;if (s == "int32") return DataType::INT32;if (s == "int64") return DataType::INT64;if (s == "fp16") return DataType::FP16;if (s == "float32") return DataType::FLOAT32;if (s == "float64") return DataType::FLOAT64;return DataType::UNKNOWN;
}int dtypeBits(DataType dt) {switch (dt) {case DataType::UINT4: return 4;case DataType::UINT8: return 8;case DataType::UINT12: return 12;case DataType::UINT16: return 16;case DataType::UINT32: return 32;case DataType::UINT64: return 64;case DataType::INT8: return 8;case DataType::INT16: return 16;case DataType::INT32: return 32;case DataType::INT64: return 64;case DataType::FP16: return 16;case DataType::FLOAT32: return 32;case DataType::FLOAT64: return 64;default: return 8;}
}struct Shape {int n = 1;int d = 1;int h = 1;int w = 1;int c = 1;bool provided = false;size_t totalSize() const {return static_cast<size_t>(n) * d * h * w * 8;}std::string toString() const {return "{" + std::to_string(n) + "," + std::to_string(d) + "," +std::to_string(h) + "," + std::to_string(w) + "," +std::to_string(c) + "}";}
};struct CompareResult {size_t index;int n, d, h, w, c;double val1;double val2;double diff;
};// Read raw bytes from file
std::vector<uint8_t> readRawFile(const std::string& path) {std::ifstream file(path, std::ios::binary | std::ios::ate);if (!file) {throw std::runtime_error("Cannot open file: " + path);}size_t size = file.tellg();file.seekg(0, std::ios::beg);std::vector<uint8_t> data(size);if (!file.read(reinterpret_cast<char*>(data.data()), size)) {throw std::runtime_error("Cannot read file: " + path);}return data;
}// Convert raw bytes to double values based on dtype
std::vector<double> convertToDouble(const std::vector<uint8_t>& raw, DataType dtype) {std::vector<double> result;switch (dtype) {case DataType::UINT4: {// 2 values per byteresult.reserve(raw.size() * 2);for (uint8_t b : raw) {result.push_back(static_cast<double>(b & 0x0F));result.push_back(static_cast<double>((b >> 4) & 0x0F));}break;}case DataType::UINT8: {result.reserve(raw.size());for (uint8_t b : raw) {result.push_back(static_cast<double>(b));}break;}case DataType::INT8: {result.reserve(raw.size());for (uint8_t b : raw) {result.push_back(static_cast<double>(static_cast<int8_t>(b)));}break;}case DataType::UINT12: {// 2 values per 3 bytesresult.reserve((raw.size() * 2) / 3);for (size_t i = 0; i + 2 < raw.size(); i += 3) {uint32_t v1 = raw[i] | ((raw[i + 1] & 0x0F) << 8);uint32_t v2 = ((raw[i + 1] >> 4) & 0x0F) | (raw[i + 2] << 4);result.push_back(static_cast<double>(v1));result.push_back(static_cast<double>(v2));}break;}case DataType::UINT16: {result.reserve(raw.size() / 2);for (size_t i = 0; i + 1 < raw.size(); i += 2) {uint16_t v = raw[i] | (raw[i + 1] << 8);result.push_back(static_cast<double>(v));}break;}case DataType::UINT32: {result.reserve(raw.size() / 4);for (size_t i = 0; i + 3 < raw.size(); i += 4) {uint32_t v = static_cast<uint32_t>(raw[i]) |(static_cast<uint32_t>(raw[i + 1]) << 8) |(static_cast<uint32_t>(raw[i + 2]) << 16) |(static_cast<uint32_t>(raw[i + 3]) << 24);result.push_back(static_cast<double>(v));}break;}case DataType::UINT64: {result.reserve(raw.size() / 8);for (size_t i = 0; i + 7 < raw.size(); i += 8) {uint64_t v = 0;for (int j = 0; j < 8; j++) {v |= (static_cast<uint64_t>(raw[i + j]) << (j * 8));}result.push_back(static_cast<double>(v));}break;}case DataType::INT16: {result.reserve(raw.size() / 2);for (size_t i = 0; i + 1 < raw.size(); i += 2) {uint16_t uv = static_cast<uint16_t>(raw[i]) |(static_cast<uint16_t>(raw[i + 1]) << 8);result.push_back(static_cast<double>(static_cast<int16_t>(uv)));}break;}case DataType::INT32: {result.reserve(raw.size() / 4);for (size_t i = 0; i + 3 < raw.size(); i += 4) {uint32_t uv = static_cast<uint32_t>(raw[i]) |(static_cast<uint32_t>(raw[i + 1]) << 8) |(static_cast<uint32_t>(raw[i + 2]) << 16) |(static_cast<uint32_t>(raw[i + 3]) << 24);result.push_back(static_cast<double>(static_cast<int32_t>(uv)));}break;}case DataType::INT64: {result.reserve(raw.size() / 8);for (size_t i = 0; i + 7 < raw.size(); i += 8) {uint64_t uv = 0;for (int j = 0; j < 8; j++) {uv |= (static_cast<uint64_t>(raw[i + j]) << (j * 8));}result.push_back(static_cast<double>(static_cast<int64_t>(uv)));}break;}case DataType::FP16: {result.reserve(raw.size() / 2);for (size_t i = 0; i + 1 < raw.size(); i += 2) {uint16_t v = raw[i] | (raw[i + 1] << 8);result.push_back(static_cast<double>(fp16ToFloat(v)));}break;}case DataType::FLOAT32: {result.reserve(raw.size() / 4);for (size_t i = 0; i + 3 < raw.size(); i += 4) {uint32_t v = static_cast<uint32_t>(raw[i]) |(static_cast<uint32_t>(raw[i + 1]) << 8) |(static_cast<uint32_t>(raw[i + 2]) << 16) |(static_cast<uint32_t>(raw[i + 3]) << 24);result.push_back(static_cast<double>(*reinterpret_cast<float*>(&v)));}break;}case DataType::FLOAT64: {result.reserve(raw.size() / 8);for (size_t i = 0; i + 7 < raw.size(); i += 8) {uint64_t v = 0;for (int j = 0; j < 8; j++) {v |= (static_cast<uint64_t>(raw[i + j]) << (j * 8));}result.push_back(*reinterpret_cast<double*>(&v));}break;}default:throw std::runtime_error("Unknown data type");}return result;
}void exportToTxt(const std::vector<double>& data, const std::string& outputPath) {std::ofstream out(outputPath);if (!out) {throw std::runtime_error("Cannot create file: " + outputPath);}for (size_t i = 0; i < data.size(); ++i) {out << std::setprecision(10) << data[i];if (i < data.size() - 1) out << "\n";}
}void getIndices(size_t index, const Shape& shape, int& n, int& d, int& h, int& w, int& c) {size_t temp = index;c = temp % shape.c; temp /= shape.c;w = temp % shape.w; temp /= shape.w;h = temp % shape.h; temp /= shape.h;d = temp % shape.d; temp /= shape.d;n = temp % shape.n;
}std::string formatIndex(size_t index, const Shape& shape) {if (shape.provided) {int n, d, h, w, c;getIndices(index, shape, n, d, h, w, c);return "[" + std::to_string(n) + "," + std::to_string(d) + "," +std::to_string(h) + "," + std::to_string(w) + "," +std::to_string(c) + "]";} else {return "[" + std::to_string(index) + "]";}
}CompareResult findFirstDiff(const std::vector<double>& data1, const std::vector<double>& data2, const Shape& shape) {size_t minSize = std::min(data1.size(), data2.size());for (size_t i = 0; i < minSize; ++i) {if (data1[i] != data2[i]) {CompareResult result;result.index = i;getIndices(i, shape, result.n, result.d, result.h, result.w, result.c);result.val1 = data1[i];result.val2 = data2[i];result.diff = std::abs(result.val1 - result.val2);return result;}}CompareResult result = {0, 0, 0, 0, 0, 0, 0.0, 0.0, 0.0};if (data1.size() != data2.size()) {result.index = minSize;result.diff = -1;}return result;
}CompareResult findMaxDiffThreaded(const std::vector<double>& data1, const std::vector<double>& data2,const Shape& shape, int numThreads) {size_t size = std::min(data1.size(), data2.size());std::vector<std::thread> threads;std::vector<CompareResult> localMax(numThreads);auto worker = [&](int threadId, size_t start, size_t end) {double maxDiff = 0.0;size_t maxIndex = 0;double maxVal1 = 0.0, maxVal2 = 0.0;for (size_t i = start; i < end; ++i) {double v1 = data1[i];double v2 = data2[i];double diff = std::abs(v1 - v2);if (diff > maxDiff) {maxDiff = diff;maxIndex = i;maxVal1 = v1;maxVal2 = v2;}}localMax[threadId].index = maxIndex;localMax[threadId].diff = maxDiff;localMax[threadId].val1 = maxVal1;localMax[threadId].val2 = maxVal2;};size_t chunkSize = size / numThreads;for (int i = 0; i < numThreads; ++i) {size_t start = i * chunkSize;size_t end = (i == numThreads - 1) ? size : (i + 1) * chunkSize;threads.emplace_back(worker, i, start, end);}for (auto& t : threads) {t.join();}CompareResult globalMax = localMax[0];for (int i = 1; i < numThreads; ++i) {if (localMax[i].diff > globalMax.diff) {globalMax = localMax[i];}}getIndices(globalMax.index, shape, globalMax.n, globalMax.d, globalMax.h, globalMax.w, globalMax.c);return globalMax;
}void generateCompareResult(const std::vector<double>& data1, const std::vector<double>& data2,const Shape& shape, const std::string& outputPath,const CompareResult& firstDiffResult,const CompareResult& maxDiffResult, int numThreads, bool isFloat) {std::ofstream out(outputPath);if (!out) {throw std::runtime_error("Cannot create file: " + outputPath);}size_t size = std::min(data1.size(), data2.size());// Determine column widths based on max valuesdouble maxVal = 0;double maxDiffVal = 0;for (size_t i = 0; i < size; ++i) {maxVal = std::max(maxVal, std::max(std::abs(data1[i]), std::abs(data2[i])));maxDiffVal = std::max(maxDiffVal, std::abs(data1[i] - data2[i]));}maxVal = std::max(maxVal, std::max(std::abs(maxDiffResult.val1), std::abs(maxDiffResult.val2)));maxDiffVal = std::max(maxDiffVal, maxDiffResult.diff);// Calculate width needed for valuesint valWidth, diffWidth;if (isFloat) {// For floats, use fixed width to show precisionvalWidth = 15;diffWidth = 15;} else {valWidth = std::max(11, static_cast<int>(std::to_string(static_cast<int64_t>(maxVal)).length()) + 2);diffWidth = std::max(11, static_cast<int>(std::to_string(static_cast<int64_t>(maxDiffVal)).length()) + 2);}int idxWidth = std::max(6, static_cast<int>(std::to_string(size).length()) + 2);// If shape provided, we need an extra column for positionbool hasShape = shape.provided;int posWidth = 17;auto printSeparator = [&]() {out << std::string(idxWidth + 3, '-');out << std::string(valWidth + 2, '-');out << std::string(valWidth + 2, '-');out << std::string(diffWidth + 2, '-');if (hasShape) out << std::string(posWidth + 2, '-');out << "\n";};auto centerText = [](const std::string& text, int width) -> std::string {if (text.length() >= static_cast<size_t>(width)) return text;int padding = width - text.length();int left = padding / 2;int right = padding - left;return std::string(left, ' ') + text + std::string(right, ' ');};auto printHeader = [&]() {out << "| " << std::setw(idxWidth) << "index" << " |"<< std::setw(valWidth) << "value1" << " |"<< std::setw(valWidth) << "value2" << " |"<< std::setw(diffWidth) << "abs(v1-v2)" << " |";if (hasShape) out << centerText("position", posWidth) << " |";out << "\n";};auto printRow = [&](size_t idx, double v1, double v2, double diff, const std::string& pos = "") {out << "| " << std::setw(idxWidth) << idx << " |";if (isFloat) {out << std::setw(valWidth) << std::setprecision(6) << std::fixed << v1 << " |"<< std::setw(valWidth) << std::setprecision(6) << std::fixed << v2 << " |"<< std::setw(diffWidth) << std::setprecision(6) << std::fixed << diff << " |";} else {out << std::setw(valWidth) << static_cast<int64_t>(v1) << " |"<< std::setw(valWidth) << static_cast<int64_t>(v2) << " |"<< std::setw(diffWidth) << static_cast<int64_t>(diff) << " |";}if (hasShape) out << centerText(pos, posWidth) << " |";out << std::right << "\n";};// Write headerprintSeparator();printHeader();printSeparator();// Write data rowsfor (size_t i = 0; i < size; ++i) {double v1 = data1[i];double v2 = data2[i];double diff = std::abs(v1 - v2);if (hasShape) {printRow(i, v1, v2, diff, formatIndex(i, shape));} else {printRow(i, v1, v2, diff);}}// Write first diff rowprintSeparator();out << "| " << std::setw(idxWidth) << "index" << " |"<< std::setw(valWidth) << "value1" << " |"<< std::setw(valWidth) << "value2" << " |"<< std::setw(diffWidth) << "first diff" << " |";if (hasShape) out << centerText(formatIndex(firstDiffResult.index, shape), posWidth) << " |";out << "\n";out << "| " << std::setw(idxWidth) << firstDiffResult.index << " |";if (isFloat) {out << std::setw(valWidth) << std::setprecision(6) << std::fixed << firstDiffResult.val1 << " |"<< std::setw(valWidth) << std::setprecision(6) << std::fixed << firstDiffResult.val2 << " |"<< std::setw(diffWidth) << std::setprecision(6) << std::fixed << firstDiffResult.diff << " |";} else {out << std::setw(valWidth) << static_cast<int64_t>(firstDiffResult.val1) << " |"<< std::setw(valWidth) << static_cast<int64_t>(firstDiffResult.val2) << " |"<< std::setw(diffWidth) << static_cast<int64_t>(firstDiffResult.diff) << " |";}if (hasShape) out << centerText(formatIndex(firstDiffResult.index, shape), posWidth) << " |";out << std::right << "\n";// Write max diff rowprintSeparator();out << "| " << std::setw(idxWidth) << "index" << " |"<< std::setw(valWidth) << "value1" << " |"<< std::setw(valWidth) << "value2" << " |"<< std::setw(diffWidth) << "max diff" << " |";if (hasShape) out << centerText(formatIndex(maxDiffResult.index, shape), posWidth) << " |";out << "\n";out << "| " << std::setw(idxWidth) << maxDiffResult.index << " |";if (isFloat) {out << std::setw(valWidth) << std::setprecision(6) << std::fixed << maxDiffResult.val1 << " |"<< std::setw(valWidth) << std::setprecision(6) << std::fixed << maxDiffResult.val2 << " |"<< std::setw(diffWidth) << std::setprecision(6) << std::fixed << maxDiffResult.diff << " |";} else {out << std::setw(valWidth) << static_cast<int64_t>(maxDiffResult.val1) << " |"<< std::setw(valWidth) << static_cast<int64_t>(maxDiffResult.val2) << " |"<< std::setw(diffWidth) << static_cast<int64_t>(maxDiffResult.diff) << " |";}if (hasShape) out << centerText(formatIndex(maxDiffResult.index, shape), posWidth) << " |";out << std::right << "\n";printSeparator();
}void printUsage(const char* prog) {std::cout << "Usage: " << prog << " <file1> <file2> [options]\n";std::cout << "Options:\n";std::cout << " --dtype=TYPE Data type: uint4, uint8, uint12, uint16, uint32, uint64,\n";std::cout << " int8, int16, int32, int64,\n";std::cout << " fp16, float32, float64 (default: uint8)\n";std::cout << " --shape=N,D,H,W,C8 Shape dimensions, default={1,size}\n";std::cout << " --threads=N Number of threads, default=auto\n";std::cout << "\nExamples:\n";std::cout << " " << prog << " a.bin b.bin\n";std::cout << " " << prog << " a.bin b.bin --dtype=float32\n";std::cout << " " << prog << " a.bin b.bin --dtype=uint16 --shape=1,3,224,224,8\n";
}int parseArgs(int argc, char* argv[], std::string& file1, std::string& file2,DataType& dtype, Shape& shape, int& numThreads) {if (argc < 3) {printUsage(argv[0]);return 1;}file1 = argv[1];file2 = argv[2];dtype = DataType::UINT8;numThreads = std::thread::hardware_concurrency();if (numThreads == 0) numThreads = 4;for (int i = 3; i < argc; ++i) {std::string arg = argv[i];if (arg.find("--dtype=") == 0) {std::string typeStr = arg.substr(8);dtype = parseDataType(typeStr);if (dtype == DataType::UNKNOWN) {std::cerr << "Error: unknown data type: " << typeStr << "\n";std::cerr << "Supported types: uint4, uint8, uint12, uint16, uint32, uint64, int8, int16, int32, int64, fp16, float32, float64\n";return 1;}} else if (arg.find("--shape=") == 0) {std::string s = arg.substr(8);std::replace(s.begin(), s.end(), ',', ' ');std::istringstream iss(s);if (!(iss >> shape.n >> shape.d >> shape.h >> shape.w >> shape.c)) {std::cerr << "Error: shape format must be N,D,H,W,C8\n";return 1;}shape.provided = true;} else if (arg.find("--threads=") == 0) {numThreads = std::stoi(arg.substr(10));} else {std::cerr << "Unknown option: " << arg << "\n";return 1;}}return 0;
}std::string getParentDirName(const std::string& filePath) {fs::path p(filePath);fs::path parent = p.parent_path();if (parent.empty()) {return p.stem().string();}return parent.filename().string();
}std::string getUniqueExportPath(const fs::path& tmpDir, const std::string& baseName, int suffix) {if (suffix == 0) {return (tmpDir / (baseName + ".txt")).string();}return (tmpDir / (baseName + "_" + std::to_string(suffix) + ".txt")).string();
}void getDataValue(const std::string& file, std::vector<double>& data, DataType dtype){std::vector<uint8_t> raw = readRawFile(file);data = convertToDouble(raw, dtype);
}void process(const std::string& file1, const std::string& file2,DataType dtype, const Shape& shape, int numThreads) {bool isFloat = (dtype == DataType::FP16 || dtype == DataType::FLOAT32 || dtype == DataType::FLOAT64);std::vector<double> data1;std::vector<double> data2;std::thread get_data_value1(getDataValue, std::ref(file1), std::ref(data1), dtype);std::thread get_data_value2(getDataValue, std::ref(file2), std::ref(data2), dtype);get_data_value1.join();get_data_value2.join();std::cout << "File1 size: " << data1.size() << " elements\n";std::cout << "File2 size: " << data2.size() << " elements\n";if (data1.size() != data2.size()) {std::cout << "Warning: File sizes differ!\n";}Shape actualShape = shape;if (!shape.provided) {actualShape.w = static_cast<int>(std::min(data1.size(), data2.size()));actualShape.provided = false;}std::cout << "\nFinding first difference...\n";auto firstDiff = findFirstDiff(data1, data2, actualShape);if (firstDiff.diff < 0) {std::cout << "Files have different sizes. First extra index: "<< formatIndex(firstDiff.index, actualShape) << "\n";} else if (firstDiff.diff > 0) {std::cout << "First difference at: " << formatIndex(firstDiff.index, actualShape) << "\n";std::cout << " File1: " << firstDiff.val1 << "\n";std::cout << " File2: " << firstDiff.val2 << "\n";std::cout << " Diff: " << firstDiff.diff << "\n";} else {std::cout << "Files are identical!\n";}// Create tmp directoryfs::path tmpDir = fs::current_path() / "tmp";fs::create_directories(tmpDir);std::cout << "\nExporting to text files...\n";std::string parentName1 = getParentDirName(file1);std::string parentName2 = getParentDirName(file2);std::string txt1 = getUniqueExportPath(tmpDir, parentName1, 0);std::string txt2;if (parentName1 == parentName2) {txt2 = getUniqueExportPath(tmpDir, parentName2, 1);} else {txt2 = getUniqueExportPath(tmpDir, parentName2, 0);}std::thread export_to_txt1(exportToTxt, data1, txt1);std::thread export_to_txt2(exportToTxt, data2, txt2);export_to_txt1.detach();export_to_txt2.detach();std::cout << "\nFinding maximum difference using " << numThreads << " threads...\n";auto maxDiff = findMaxDiffThreaded(data1, data2, actualShape, numThreads);std::cout << "Max difference at: " << formatIndex(maxDiff.index, actualShape) << "\n";std::cout << " File1: " << maxDiff.val1 << "\n";std::cout << " File2: " << maxDiff.val2 << "\n";std::cout << " Diff: " << maxDiff.diff << "\n";std::cout << "\nGenerating compareResult.txt...\n";generateCompareResult(data1, data2, actualShape, "compareResult.txt", firstDiff, maxDiff, numThreads, isFloat);std::cout << "Done!\n";
}int main(int argc, char* argv[]) {std::string file1, file2;DataType dtype;Shape shape;int numThreads;if (parseArgs(argc, argv, file1, file2, dtype, shape, numThreads) != 0) {return 1;}try {std::cout << "=== Binary Compare Tool ===\n";std::cout << "File1: " << file1 << "\n";std::cout << "File2: " << file2 << "\n";std::cout << "DataType: " << dtypeToString(dtype) << "\n";if (shape.provided) {std::cout << "Shape: " << shape.toString() << "\n";}std::cout << "Threads: " << numThreads << "\n\n";process(file1, file2, dtype, shape, numThreads);} catch (const std::exception& e) {std::cerr << "Error: " << e.what() << "\n";return 1;}return 0;
}
比较2个二进制文件并输出指定格式的txt
生命依靠吸收负熵，避免了趋向平衡的衰退