diff -pruN 1.4-2/3rd-party/hash-library/crc32.cpp 2.5-1/3rd-party/hash-library/crc32.cpp --- 1.4-2/3rd-party/hash-library/crc32.cpp 1970-01-01 00:00:00.000000000 +0000 +++ 2.5-1/3rd-party/hash-library/crc32.cpp 2022-06-15 07:33:00.000000000 +0000 @@ -0,0 +1,431 @@ +// ////////////////////////////////////////////////////////// +// crc32.cpp +// Copyright (c) 2014,2015 Stephan Brumme. All rights reserved. +// see http://create.stephan-brumme.com/disclaimer.html +// + +#include "crc32.h" + +// big endian architectures need #define __BYTE_ORDER __BIG_ENDIAN +#ifndef _MSC_VER +#include +#endif + + +/// same as reset() +CRC32::CRC32() +{ + reset(); +} + + +/// restart +void CRC32::reset() +{ + m_hash = 0; +} + + +namespace +{ + /// look-up table + static const uint32_t crc32Lookup[8][256] = + { + // generated by: + //for (uint32_t i = 0; i <= 0xFF; i++) + //{ + // uint32_t crc = i; + // for (unsigned int j = 0; j < 8; j++) + // crc = (crc >> 1) ^ ((crc & 1) * Polynomial); + // crc32Lookup[0][i] = crc; + //} + // slicing-by-8 algorithm (from Intel): + // http://www.intel.com/technology/comms/perfnet/download/CRC_generators.pdf + // http://sourceforge.net/projects/slicing-by-8/ + //for (unsigned int i = 0; i <= 0xFF; i++) + //{ + // crc32Lookup[1][i] = (crc32Lookup[0][i] >> 8) ^ crc32Lookup[0][crc32Lookup[0][i] & 0xFF]; + // crc32Lookup[2][i] = (crc32Lookup[1][i] >> 8) ^ crc32Lookup[0][crc32Lookup[1][i] & 0xFF]; + // crc32Lookup[3][i] = (crc32Lookup[2][i] >> 8) ^ crc32Lookup[0][crc32Lookup[2][i] & 0xFF]; + + // crc32Lookup[4][i] = (crc32Lookup[3][i] >> 8) ^ crc32Lookup[0][crc32Lookup[3][i] & 0xFF]; + // crc32Lookup[5][i] = (crc32Lookup[4][i] >> 8) ^ crc32Lookup[0][crc32Lookup[4][i] & 0xFF]; + // crc32Lookup[6][i] = (crc32Lookup[5][i] >> 8) ^ crc32Lookup[0][crc32Lookup[5][i] & 0xFF]; + // crc32Lookup[7][i] = (crc32Lookup[6][i] >> 8) ^ crc32Lookup[0][crc32Lookup[6][i] & 0xFF]; + //} + { 0x00000000,0x77073096,0xEE0E612C,0x990951BA,0x076DC419,0x706AF48F,0xE963A535,0x9E6495A3, + 0x0EDB8832,0x79DCB8A4,0xE0D5E91E,0x97D2D988,0x09B64C2B,0x7EB17CBD,0xE7B82D07,0x90BF1D91, + 0x1DB71064,0x6AB020F2,0xF3B97148,0x84BE41DE,0x1ADAD47D,0x6DDDE4EB,0xF4D4B551,0x83D385C7, + 0x136C9856,0x646BA8C0,0xFD62F97A,0x8A65C9EC,0x14015C4F,0x63066CD9,0xFA0F3D63,0x8D080DF5, + 0x3B6E20C8,0x4C69105E,0xD56041E4,0xA2677172,0x3C03E4D1,0x4B04D447,0xD20D85FD,0xA50AB56B, + 0x35B5A8FA,0x42B2986C,0xDBBBC9D6,0xACBCF940,0x32D86CE3,0x45DF5C75,0xDCD60DCF,0xABD13D59, + 0x26D930AC,0x51DE003A,0xC8D75180,0xBFD06116,0x21B4F4B5,0x56B3C423,0xCFBA9599,0xB8BDA50F, + 0x2802B89E,0x5F058808,0xC60CD9B2,0xB10BE924,0x2F6F7C87,0x58684C11,0xC1611DAB,0xB6662D3D, + 0x76DC4190,0x01DB7106,0x98D220BC,0xEFD5102A,0x71B18589,0x06B6B51F,0x9FBFE4A5,0xE8B8D433, + 0x7807C9A2,0x0F00F934,0x9609A88E,0xE10E9818,0x7F6A0DBB,0x086D3D2D,0x91646C97,0xE6635C01, + 0x6B6B51F4,0x1C6C6162,0x856530D8,0xF262004E,0x6C0695ED,0x1B01A57B,0x8208F4C1,0xF50FC457, + 0x65B0D9C6,0x12B7E950,0x8BBEB8EA,0xFCB9887C,0x62DD1DDF,0x15DA2D49,0x8CD37CF3,0xFBD44C65, + 0x4DB26158,0x3AB551CE,0xA3BC0074,0xD4BB30E2,0x4ADFA541,0x3DD895D7,0xA4D1C46D,0xD3D6F4FB, + 0x4369E96A,0x346ED9FC,0xAD678846,0xDA60B8D0,0x44042D73,0x33031DE5,0xAA0A4C5F,0xDD0D7CC9, + 0x5005713C,0x270241AA,0xBE0B1010,0xC90C2086,0x5768B525,0x206F85B3,0xB966D409,0xCE61E49F, + 0x5EDEF90E,0x29D9C998,0xB0D09822,0xC7D7A8B4,0x59B33D17,0x2EB40D81,0xB7BD5C3B,0xC0BA6CAD, + 0xEDB88320,0x9ABFB3B6,0x03B6E20C,0x74B1D29A,0xEAD54739,0x9DD277AF,0x04DB2615,0x73DC1683, + 0xE3630B12,0x94643B84,0x0D6D6A3E,0x7A6A5AA8,0xE40ECF0B,0x9309FF9D,0x0A00AE27,0x7D079EB1, + 0xF00F9344,0x8708A3D2,0x1E01F268,0x6906C2FE,0xF762575D,0x806567CB,0x196C3671,0x6E6B06E7, + 0xFED41B76,0x89D32BE0,0x10DA7A5A,0x67DD4ACC,0xF9B9DF6F,0x8EBEEFF9,0x17B7BE43,0x60B08ED5, + 0xD6D6A3E8,0xA1D1937E,0x38D8C2C4,0x4FDFF252,0xD1BB67F1,0xA6BC5767,0x3FB506DD,0x48B2364B, + 0xD80D2BDA,0xAF0A1B4C,0x36034AF6,0x41047A60,0xDF60EFC3,0xA867DF55,0x316E8EEF,0x4669BE79, + 0xCB61B38C,0xBC66831A,0x256FD2A0,0x5268E236,0xCC0C7795,0xBB0B4703,0x220216B9,0x5505262F, + 0xC5BA3BBE,0xB2BD0B28,0x2BB45A92,0x5CB36A04,0xC2D7FFA7,0xB5D0CF31,0x2CD99E8B,0x5BDEAE1D, + 0x9B64C2B0,0xEC63F226,0x756AA39C,0x026D930A,0x9C0906A9,0xEB0E363F,0x72076785,0x05005713, + 0x95BF4A82,0xE2B87A14,0x7BB12BAE,0x0CB61B38,0x92D28E9B,0xE5D5BE0D,0x7CDCEFB7,0x0BDBDF21, + 0x86D3D2D4,0xF1D4E242,0x68DDB3F8,0x1FDA836E,0x81BE16CD,0xF6B9265B,0x6FB077E1,0x18B74777, + 0x88085AE6,0xFF0F6A70,0x66063BCA,0x11010B5C,0x8F659EFF,0xF862AE69,0x616BFFD3,0x166CCF45, + 0xA00AE278,0xD70DD2EE,0x4E048354,0x3903B3C2,0xA7672661,0xD06016F7,0x4969474D,0x3E6E77DB, + 0xAED16A4A,0xD9D65ADC,0x40DF0B66,0x37D83BF0,0xA9BCAE53,0xDEBB9EC5,0x47B2CF7F,0x30B5FFE9, + 0xBDBDF21C,0xCABAC28A,0x53B39330,0x24B4A3A6,0xBAD03605,0xCDD70693,0x54DE5729,0x23D967BF, + 0xB3667A2E,0xC4614AB8,0x5D681B02,0x2A6F2B94,0xB40BBE37,0xC30C8EA1,0x5A05DF1B,0x2D02EF8D }, + + { 0x00000000,0x191B3141,0x32366282,0x2B2D53C3,0x646CC504,0x7D77F445,0x565AA786,0x4F4196C7, + 0xC8D98A08,0xD1C2BB49,0xFAEFE88A,0xE3F4D9CB,0xACB54F0C,0xB5AE7E4D,0x9E832D8E,0x87981CCF, + 0x4AC21251,0x53D92310,0x78F470D3,0x61EF4192,0x2EAED755,0x37B5E614,0x1C98B5D7,0x05838496, + 0x821B9859,0x9B00A918,0xB02DFADB,0xA936CB9A,0xE6775D5D,0xFF6C6C1C,0xD4413FDF,0xCD5A0E9E, + 0x958424A2,0x8C9F15E3,0xA7B24620,0xBEA97761,0xF1E8E1A6,0xE8F3D0E7,0xC3DE8324,0xDAC5B265, + 0x5D5DAEAA,0x44469FEB,0x6F6BCC28,0x7670FD69,0x39316BAE,0x202A5AEF,0x0B07092C,0x121C386D, + 0xDF4636F3,0xC65D07B2,0xED705471,0xF46B6530,0xBB2AF3F7,0xA231C2B6,0x891C9175,0x9007A034, + 0x179FBCFB,0x0E848DBA,0x25A9DE79,0x3CB2EF38,0x73F379FF,0x6AE848BE,0x41C51B7D,0x58DE2A3C, + 0xF0794F05,0xE9627E44,0xC24F2D87,0xDB541CC6,0x94158A01,0x8D0EBB40,0xA623E883,0xBF38D9C2, + 0x38A0C50D,0x21BBF44C,0x0A96A78F,0x138D96CE,0x5CCC0009,0x45D73148,0x6EFA628B,0x77E153CA, + 0xBABB5D54,0xA3A06C15,0x888D3FD6,0x91960E97,0xDED79850,0xC7CCA911,0xECE1FAD2,0xF5FACB93, + 0x7262D75C,0x6B79E61D,0x4054B5DE,0x594F849F,0x160E1258,0x0F152319,0x243870DA,0x3D23419B, + 0x65FD6BA7,0x7CE65AE6,0x57CB0925,0x4ED03864,0x0191AEA3,0x188A9FE2,0x33A7CC21,0x2ABCFD60, + 0xAD24E1AF,0xB43FD0EE,0x9F12832D,0x8609B26C,0xC94824AB,0xD05315EA,0xFB7E4629,0xE2657768, + 0x2F3F79F6,0x362448B7,0x1D091B74,0x04122A35,0x4B53BCF2,0x52488DB3,0x7965DE70,0x607EEF31, + 0xE7E6F3FE,0xFEFDC2BF,0xD5D0917C,0xCCCBA03D,0x838A36FA,0x9A9107BB,0xB1BC5478,0xA8A76539, + 0x3B83984B,0x2298A90A,0x09B5FAC9,0x10AECB88,0x5FEF5D4F,0x46F46C0E,0x6DD93FCD,0x74C20E8C, + 0xF35A1243,0xEA412302,0xC16C70C1,0xD8774180,0x9736D747,0x8E2DE606,0xA500B5C5,0xBC1B8484, + 0x71418A1A,0x685ABB5B,0x4377E898,0x5A6CD9D9,0x152D4F1E,0x0C367E5F,0x271B2D9C,0x3E001CDD, + 0xB9980012,0xA0833153,0x8BAE6290,0x92B553D1,0xDDF4C516,0xC4EFF457,0xEFC2A794,0xF6D996D5, + 0xAE07BCE9,0xB71C8DA8,0x9C31DE6B,0x852AEF2A,0xCA6B79ED,0xD37048AC,0xF85D1B6F,0xE1462A2E, + 0x66DE36E1,0x7FC507A0,0x54E85463,0x4DF36522,0x02B2F3E5,0x1BA9C2A4,0x30849167,0x299FA026, + 0xE4C5AEB8,0xFDDE9FF9,0xD6F3CC3A,0xCFE8FD7B,0x80A96BBC,0x99B25AFD,0xB29F093E,0xAB84387F, + 0x2C1C24B0,0x350715F1,0x1E2A4632,0x07317773,0x4870E1B4,0x516BD0F5,0x7A468336,0x635DB277, + 0xCBFAD74E,0xD2E1E60F,0xF9CCB5CC,0xE0D7848D,0xAF96124A,0xB68D230B,0x9DA070C8,0x84BB4189, + 0x03235D46,0x1A386C07,0x31153FC4,0x280E0E85,0x674F9842,0x7E54A903,0x5579FAC0,0x4C62CB81, + 0x8138C51F,0x9823F45E,0xB30EA79D,0xAA1596DC,0xE554001B,0xFC4F315A,0xD7626299,0xCE7953D8, + 0x49E14F17,0x50FA7E56,0x7BD72D95,0x62CC1CD4,0x2D8D8A13,0x3496BB52,0x1FBBE891,0x06A0D9D0, + 0x5E7EF3EC,0x4765C2AD,0x6C48916E,0x7553A02F,0x3A1236E8,0x230907A9,0x0824546A,0x113F652B, + 0x96A779E4,0x8FBC48A5,0xA4911B66,0xBD8A2A27,0xF2CBBCE0,0xEBD08DA1,0xC0FDDE62,0xD9E6EF23, + 0x14BCE1BD,0x0DA7D0FC,0x268A833F,0x3F91B27E,0x70D024B9,0x69CB15F8,0x42E6463B,0x5BFD777A, + 0xDC656BB5,0xC57E5AF4,0xEE530937,0xF7483876,0xB809AEB1,0xA1129FF0,0x8A3FCC33,0x9324FD72 }, + + { 0x00000000,0x01C26A37,0x0384D46E,0x0246BE59,0x0709A8DC,0x06CBC2EB,0x048D7CB2,0x054F1685, + 0x0E1351B8,0x0FD13B8F,0x0D9785D6,0x0C55EFE1,0x091AF964,0x08D89353,0x0A9E2D0A,0x0B5C473D, + 0x1C26A370,0x1DE4C947,0x1FA2771E,0x1E601D29,0x1B2F0BAC,0x1AED619B,0x18ABDFC2,0x1969B5F5, + 0x1235F2C8,0x13F798FF,0x11B126A6,0x10734C91,0x153C5A14,0x14FE3023,0x16B88E7A,0x177AE44D, + 0x384D46E0,0x398F2CD7,0x3BC9928E,0x3A0BF8B9,0x3F44EE3C,0x3E86840B,0x3CC03A52,0x3D025065, + 0x365E1758,0x379C7D6F,0x35DAC336,0x3418A901,0x3157BF84,0x3095D5B3,0x32D36BEA,0x331101DD, + 0x246BE590,0x25A98FA7,0x27EF31FE,0x262D5BC9,0x23624D4C,0x22A0277B,0x20E69922,0x2124F315, + 0x2A78B428,0x2BBADE1F,0x29FC6046,0x283E0A71,0x2D711CF4,0x2CB376C3,0x2EF5C89A,0x2F37A2AD, + 0x709A8DC0,0x7158E7F7,0x731E59AE,0x72DC3399,0x7793251C,0x76514F2B,0x7417F172,0x75D59B45, + 0x7E89DC78,0x7F4BB64F,0x7D0D0816,0x7CCF6221,0x798074A4,0x78421E93,0x7A04A0CA,0x7BC6CAFD, + 0x6CBC2EB0,0x6D7E4487,0x6F38FADE,0x6EFA90E9,0x6BB5866C,0x6A77EC5B,0x68315202,0x69F33835, + 0x62AF7F08,0x636D153F,0x612BAB66,0x60E9C151,0x65A6D7D4,0x6464BDE3,0x662203BA,0x67E0698D, + 0x48D7CB20,0x4915A117,0x4B531F4E,0x4A917579,0x4FDE63FC,0x4E1C09CB,0x4C5AB792,0x4D98DDA5, + 0x46C49A98,0x4706F0AF,0x45404EF6,0x448224C1,0x41CD3244,0x400F5873,0x4249E62A,0x438B8C1D, + 0x54F16850,0x55330267,0x5775BC3E,0x56B7D609,0x53F8C08C,0x523AAABB,0x507C14E2,0x51BE7ED5, + 0x5AE239E8,0x5B2053DF,0x5966ED86,0x58A487B1,0x5DEB9134,0x5C29FB03,0x5E6F455A,0x5FAD2F6D, + 0xE1351B80,0xE0F771B7,0xE2B1CFEE,0xE373A5D9,0xE63CB35C,0xE7FED96B,0xE5B86732,0xE47A0D05, + 0xEF264A38,0xEEE4200F,0xECA29E56,0xED60F461,0xE82FE2E4,0xE9ED88D3,0xEBAB368A,0xEA695CBD, + 0xFD13B8F0,0xFCD1D2C7,0xFE976C9E,0xFF5506A9,0xFA1A102C,0xFBD87A1B,0xF99EC442,0xF85CAE75, + 0xF300E948,0xF2C2837F,0xF0843D26,0xF1465711,0xF4094194,0xF5CB2BA3,0xF78D95FA,0xF64FFFCD, + 0xD9785D60,0xD8BA3757,0xDAFC890E,0xDB3EE339,0xDE71F5BC,0xDFB39F8B,0xDDF521D2,0xDC374BE5, + 0xD76B0CD8,0xD6A966EF,0xD4EFD8B6,0xD52DB281,0xD062A404,0xD1A0CE33,0xD3E6706A,0xD2241A5D, + 0xC55EFE10,0xC49C9427,0xC6DA2A7E,0xC7184049,0xC25756CC,0xC3953CFB,0xC1D382A2,0xC011E895, + 0xCB4DAFA8,0xCA8FC59F,0xC8C97BC6,0xC90B11F1,0xCC440774,0xCD866D43,0xCFC0D31A,0xCE02B92D, + 0x91AF9640,0x906DFC77,0x922B422E,0x93E92819,0x96A63E9C,0x976454AB,0x9522EAF2,0x94E080C5, + 0x9FBCC7F8,0x9E7EADCF,0x9C381396,0x9DFA79A1,0x98B56F24,0x99770513,0x9B31BB4A,0x9AF3D17D, + 0x8D893530,0x8C4B5F07,0x8E0DE15E,0x8FCF8B69,0x8A809DEC,0x8B42F7DB,0x89044982,0x88C623B5, + 0x839A6488,0x82580EBF,0x801EB0E6,0x81DCDAD1,0x8493CC54,0x8551A663,0x8717183A,0x86D5720D, + 0xA9E2D0A0,0xA820BA97,0xAA6604CE,0xABA46EF9,0xAEEB787C,0xAF29124B,0xAD6FAC12,0xACADC625, + 0xA7F18118,0xA633EB2F,0xA4755576,0xA5B73F41,0xA0F829C4,0xA13A43F3,0xA37CFDAA,0xA2BE979D, + 0xB5C473D0,0xB40619E7,0xB640A7BE,0xB782CD89,0xB2CDDB0C,0xB30FB13B,0xB1490F62,0xB08B6555, + 0xBBD72268,0xBA15485F,0xB853F606,0xB9919C31,0xBCDE8AB4,0xBD1CE083,0xBF5A5EDA,0xBE9834ED }, + + { 0x00000000,0xB8BC6765,0xAA09C88B,0x12B5AFEE,0x8F629757,0x37DEF032,0x256B5FDC,0x9DD738B9, + 0xC5B428EF,0x7D084F8A,0x6FBDE064,0xD7018701,0x4AD6BFB8,0xF26AD8DD,0xE0DF7733,0x58631056, + 0x5019579F,0xE8A530FA,0xFA109F14,0x42ACF871,0xDF7BC0C8,0x67C7A7AD,0x75720843,0xCDCE6F26, + 0x95AD7F70,0x2D111815,0x3FA4B7FB,0x8718D09E,0x1ACFE827,0xA2738F42,0xB0C620AC,0x087A47C9, + 0xA032AF3E,0x188EC85B,0x0A3B67B5,0xB28700D0,0x2F503869,0x97EC5F0C,0x8559F0E2,0x3DE59787, + 0x658687D1,0xDD3AE0B4,0xCF8F4F5A,0x7733283F,0xEAE41086,0x525877E3,0x40EDD80D,0xF851BF68, + 0xF02BF8A1,0x48979FC4,0x5A22302A,0xE29E574F,0x7F496FF6,0xC7F50893,0xD540A77D,0x6DFCC018, + 0x359FD04E,0x8D23B72B,0x9F9618C5,0x272A7FA0,0xBAFD4719,0x0241207C,0x10F48F92,0xA848E8F7, + 0x9B14583D,0x23A83F58,0x311D90B6,0x89A1F7D3,0x1476CF6A,0xACCAA80F,0xBE7F07E1,0x06C36084, + 0x5EA070D2,0xE61C17B7,0xF4A9B859,0x4C15DF3C,0xD1C2E785,0x697E80E0,0x7BCB2F0E,0xC377486B, + 0xCB0D0FA2,0x73B168C7,0x6104C729,0xD9B8A04C,0x446F98F5,0xFCD3FF90,0xEE66507E,0x56DA371B, + 0x0EB9274D,0xB6054028,0xA4B0EFC6,0x1C0C88A3,0x81DBB01A,0x3967D77F,0x2BD27891,0x936E1FF4, + 0x3B26F703,0x839A9066,0x912F3F88,0x299358ED,0xB4446054,0x0CF80731,0x1E4DA8DF,0xA6F1CFBA, + 0xFE92DFEC,0x462EB889,0x549B1767,0xEC277002,0x71F048BB,0xC94C2FDE,0xDBF98030,0x6345E755, + 0x6B3FA09C,0xD383C7F9,0xC1366817,0x798A0F72,0xE45D37CB,0x5CE150AE,0x4E54FF40,0xF6E89825, + 0xAE8B8873,0x1637EF16,0x048240F8,0xBC3E279D,0x21E91F24,0x99557841,0x8BE0D7AF,0x335CB0CA, + 0xED59B63B,0x55E5D15E,0x47507EB0,0xFFEC19D5,0x623B216C,0xDA874609,0xC832E9E7,0x708E8E82, + 0x28ED9ED4,0x9051F9B1,0x82E4565F,0x3A58313A,0xA78F0983,0x1F336EE6,0x0D86C108,0xB53AA66D, + 0xBD40E1A4,0x05FC86C1,0x1749292F,0xAFF54E4A,0x322276F3,0x8A9E1196,0x982BBE78,0x2097D91D, + 0x78F4C94B,0xC048AE2E,0xD2FD01C0,0x6A4166A5,0xF7965E1C,0x4F2A3979,0x5D9F9697,0xE523F1F2, + 0x4D6B1905,0xF5D77E60,0xE762D18E,0x5FDEB6EB,0xC2098E52,0x7AB5E937,0x680046D9,0xD0BC21BC, + 0x88DF31EA,0x3063568F,0x22D6F961,0x9A6A9E04,0x07BDA6BD,0xBF01C1D8,0xADB46E36,0x15080953, + 0x1D724E9A,0xA5CE29FF,0xB77B8611,0x0FC7E174,0x9210D9CD,0x2AACBEA8,0x38191146,0x80A57623, + 0xD8C66675,0x607A0110,0x72CFAEFE,0xCA73C99B,0x57A4F122,0xEF189647,0xFDAD39A9,0x45115ECC, + 0x764DEE06,0xCEF18963,0xDC44268D,0x64F841E8,0xF92F7951,0x41931E34,0x5326B1DA,0xEB9AD6BF, + 0xB3F9C6E9,0x0B45A18C,0x19F00E62,0xA14C6907,0x3C9B51BE,0x842736DB,0x96929935,0x2E2EFE50, + 0x2654B999,0x9EE8DEFC,0x8C5D7112,0x34E11677,0xA9362ECE,0x118A49AB,0x033FE645,0xBB838120, + 0xE3E09176,0x5B5CF613,0x49E959FD,0xF1553E98,0x6C820621,0xD43E6144,0xC68BCEAA,0x7E37A9CF, + 0xD67F4138,0x6EC3265D,0x7C7689B3,0xC4CAEED6,0x591DD66F,0xE1A1B10A,0xF3141EE4,0x4BA87981, + 0x13CB69D7,0xAB770EB2,0xB9C2A15C,0x017EC639,0x9CA9FE80,0x241599E5,0x36A0360B,0x8E1C516E, + 0x866616A7,0x3EDA71C2,0x2C6FDE2C,0x94D3B949,0x090481F0,0xB1B8E695,0xA30D497B,0x1BB12E1E, + 0x43D23E48,0xFB6E592D,0xE9DBF6C3,0x516791A6,0xCCB0A91F,0x740CCE7A,0x66B96194,0xDE0506F1 }, + + { 0x00000000,0x3D6029B0,0x7AC05360,0x47A07AD0,0xF580A6C0,0xC8E08F70,0x8F40F5A0,0xB220DC10, + 0x30704BC1,0x0D106271,0x4AB018A1,0x77D03111,0xC5F0ED01,0xF890C4B1,0xBF30BE61,0x825097D1, + 0x60E09782,0x5D80BE32,0x1A20C4E2,0x2740ED52,0x95603142,0xA80018F2,0xEFA06222,0xD2C04B92, + 0x5090DC43,0x6DF0F5F3,0x2A508F23,0x1730A693,0xA5107A83,0x98705333,0xDFD029E3,0xE2B00053, + 0xC1C12F04,0xFCA106B4,0xBB017C64,0x866155D4,0x344189C4,0x0921A074,0x4E81DAA4,0x73E1F314, + 0xF1B164C5,0xCCD14D75,0x8B7137A5,0xB6111E15,0x0431C205,0x3951EBB5,0x7EF19165,0x4391B8D5, + 0xA121B886,0x9C419136,0xDBE1EBE6,0xE681C256,0x54A11E46,0x69C137F6,0x2E614D26,0x13016496, + 0x9151F347,0xAC31DAF7,0xEB91A027,0xD6F18997,0x64D15587,0x59B17C37,0x1E1106E7,0x23712F57, + 0x58F35849,0x659371F9,0x22330B29,0x1F532299,0xAD73FE89,0x9013D739,0xD7B3ADE9,0xEAD38459, + 0x68831388,0x55E33A38,0x124340E8,0x2F236958,0x9D03B548,0xA0639CF8,0xE7C3E628,0xDAA3CF98, + 0x3813CFCB,0x0573E67B,0x42D39CAB,0x7FB3B51B,0xCD93690B,0xF0F340BB,0xB7533A6B,0x8A3313DB, + 0x0863840A,0x3503ADBA,0x72A3D76A,0x4FC3FEDA,0xFDE322CA,0xC0830B7A,0x872371AA,0xBA43581A, + 0x9932774D,0xA4525EFD,0xE3F2242D,0xDE920D9D,0x6CB2D18D,0x51D2F83D,0x167282ED,0x2B12AB5D, + 0xA9423C8C,0x9422153C,0xD3826FEC,0xEEE2465C,0x5CC29A4C,0x61A2B3FC,0x2602C92C,0x1B62E09C, + 0xF9D2E0CF,0xC4B2C97F,0x8312B3AF,0xBE729A1F,0x0C52460F,0x31326FBF,0x7692156F,0x4BF23CDF, + 0xC9A2AB0E,0xF4C282BE,0xB362F86E,0x8E02D1DE,0x3C220DCE,0x0142247E,0x46E25EAE,0x7B82771E, + 0xB1E6B092,0x8C869922,0xCB26E3F2,0xF646CA42,0x44661652,0x79063FE2,0x3EA64532,0x03C66C82, + 0x8196FB53,0xBCF6D2E3,0xFB56A833,0xC6368183,0x74165D93,0x49767423,0x0ED60EF3,0x33B62743, + 0xD1062710,0xEC660EA0,0xABC67470,0x96A65DC0,0x248681D0,0x19E6A860,0x5E46D2B0,0x6326FB00, + 0xE1766CD1,0xDC164561,0x9BB63FB1,0xA6D61601,0x14F6CA11,0x2996E3A1,0x6E369971,0x5356B0C1, + 0x70279F96,0x4D47B626,0x0AE7CCF6,0x3787E546,0x85A73956,0xB8C710E6,0xFF676A36,0xC2074386, + 0x4057D457,0x7D37FDE7,0x3A978737,0x07F7AE87,0xB5D77297,0x88B75B27,0xCF1721F7,0xF2770847, + 0x10C70814,0x2DA721A4,0x6A075B74,0x576772C4,0xE547AED4,0xD8278764,0x9F87FDB4,0xA2E7D404, + 0x20B743D5,0x1DD76A65,0x5A7710B5,0x67173905,0xD537E515,0xE857CCA5,0xAFF7B675,0x92979FC5, + 0xE915E8DB,0xD475C16B,0x93D5BBBB,0xAEB5920B,0x1C954E1B,0x21F567AB,0x66551D7B,0x5B3534CB, + 0xD965A31A,0xE4058AAA,0xA3A5F07A,0x9EC5D9CA,0x2CE505DA,0x11852C6A,0x562556BA,0x6B457F0A, + 0x89F57F59,0xB49556E9,0xF3352C39,0xCE550589,0x7C75D999,0x4115F029,0x06B58AF9,0x3BD5A349, + 0xB9853498,0x84E51D28,0xC34567F8,0xFE254E48,0x4C059258,0x7165BBE8,0x36C5C138,0x0BA5E888, + 0x28D4C7DF,0x15B4EE6F,0x521494BF,0x6F74BD0F,0xDD54611F,0xE03448AF,0xA794327F,0x9AF41BCF, + 0x18A48C1E,0x25C4A5AE,0x6264DF7E,0x5F04F6CE,0xED242ADE,0xD044036E,0x97E479BE,0xAA84500E, + 0x4834505D,0x755479ED,0x32F4033D,0x0F942A8D,0xBDB4F69D,0x80D4DF2D,0xC774A5FD,0xFA148C4D, + 0x78441B9C,0x4524322C,0x028448FC,0x3FE4614C,0x8DC4BD5C,0xB0A494EC,0xF704EE3C,0xCA64C78C }, + + { 0x00000000,0xCB5CD3A5,0x4DC8A10B,0x869472AE,0x9B914216,0x50CD91B3,0xD659E31D,0x1D0530B8, + 0xEC53826D,0x270F51C8,0xA19B2366,0x6AC7F0C3,0x77C2C07B,0xBC9E13DE,0x3A0A6170,0xF156B2D5, + 0x03D6029B,0xC88AD13E,0x4E1EA390,0x85427035,0x9847408D,0x531B9328,0xD58FE186,0x1ED33223, + 0xEF8580F6,0x24D95353,0xA24D21FD,0x6911F258,0x7414C2E0,0xBF481145,0x39DC63EB,0xF280B04E, + 0x07AC0536,0xCCF0D693,0x4A64A43D,0x81387798,0x9C3D4720,0x57619485,0xD1F5E62B,0x1AA9358E, + 0xEBFF875B,0x20A354FE,0xA6372650,0x6D6BF5F5,0x706EC54D,0xBB3216E8,0x3DA66446,0xF6FAB7E3, + 0x047A07AD,0xCF26D408,0x49B2A6A6,0x82EE7503,0x9FEB45BB,0x54B7961E,0xD223E4B0,0x197F3715, + 0xE82985C0,0x23755665,0xA5E124CB,0x6EBDF76E,0x73B8C7D6,0xB8E41473,0x3E7066DD,0xF52CB578, + 0x0F580A6C,0xC404D9C9,0x4290AB67,0x89CC78C2,0x94C9487A,0x5F959BDF,0xD901E971,0x125D3AD4, + 0xE30B8801,0x28575BA4,0xAEC3290A,0x659FFAAF,0x789ACA17,0xB3C619B2,0x35526B1C,0xFE0EB8B9, + 0x0C8E08F7,0xC7D2DB52,0x4146A9FC,0x8A1A7A59,0x971F4AE1,0x5C439944,0xDAD7EBEA,0x118B384F, + 0xE0DD8A9A,0x2B81593F,0xAD152B91,0x6649F834,0x7B4CC88C,0xB0101B29,0x36846987,0xFDD8BA22, + 0x08F40F5A,0xC3A8DCFF,0x453CAE51,0x8E607DF4,0x93654D4C,0x58399EE9,0xDEADEC47,0x15F13FE2, + 0xE4A78D37,0x2FFB5E92,0xA96F2C3C,0x6233FF99,0x7F36CF21,0xB46A1C84,0x32FE6E2A,0xF9A2BD8F, + 0x0B220DC1,0xC07EDE64,0x46EAACCA,0x8DB67F6F,0x90B34FD7,0x5BEF9C72,0xDD7BEEDC,0x16273D79, + 0xE7718FAC,0x2C2D5C09,0xAAB92EA7,0x61E5FD02,0x7CE0CDBA,0xB7BC1E1F,0x31286CB1,0xFA74BF14, + 0x1EB014D8,0xD5ECC77D,0x5378B5D3,0x98246676,0x852156CE,0x4E7D856B,0xC8E9F7C5,0x03B52460, + 0xF2E396B5,0x39BF4510,0xBF2B37BE,0x7477E41B,0x6972D4A3,0xA22E0706,0x24BA75A8,0xEFE6A60D, + 0x1D661643,0xD63AC5E6,0x50AEB748,0x9BF264ED,0x86F75455,0x4DAB87F0,0xCB3FF55E,0x006326FB, + 0xF135942E,0x3A69478B,0xBCFD3525,0x77A1E680,0x6AA4D638,0xA1F8059D,0x276C7733,0xEC30A496, + 0x191C11EE,0xD240C24B,0x54D4B0E5,0x9F886340,0x828D53F8,0x49D1805D,0xCF45F2F3,0x04192156, + 0xF54F9383,0x3E134026,0xB8873288,0x73DBE12D,0x6EDED195,0xA5820230,0x2316709E,0xE84AA33B, + 0x1ACA1375,0xD196C0D0,0x5702B27E,0x9C5E61DB,0x815B5163,0x4A0782C6,0xCC93F068,0x07CF23CD, + 0xF6999118,0x3DC542BD,0xBB513013,0x700DE3B6,0x6D08D30E,0xA65400AB,0x20C07205,0xEB9CA1A0, + 0x11E81EB4,0xDAB4CD11,0x5C20BFBF,0x977C6C1A,0x8A795CA2,0x41258F07,0xC7B1FDA9,0x0CED2E0C, + 0xFDBB9CD9,0x36E74F7C,0xB0733DD2,0x7B2FEE77,0x662ADECF,0xAD760D6A,0x2BE27FC4,0xE0BEAC61, + 0x123E1C2F,0xD962CF8A,0x5FF6BD24,0x94AA6E81,0x89AF5E39,0x42F38D9C,0xC467FF32,0x0F3B2C97, + 0xFE6D9E42,0x35314DE7,0xB3A53F49,0x78F9ECEC,0x65FCDC54,0xAEA00FF1,0x28347D5F,0xE368AEFA, + 0x16441B82,0xDD18C827,0x5B8CBA89,0x90D0692C,0x8DD55994,0x46898A31,0xC01DF89F,0x0B412B3A, + 0xFA1799EF,0x314B4A4A,0xB7DF38E4,0x7C83EB41,0x6186DBF9,0xAADA085C,0x2C4E7AF2,0xE712A957, + 0x15921919,0xDECECABC,0x585AB812,0x93066BB7,0x8E035B0F,0x455F88AA,0xC3CBFA04,0x089729A1, + 0xF9C19B74,0x329D48D1,0xB4093A7F,0x7F55E9DA,0x6250D962,0xA90C0AC7,0x2F987869,0xE4C4ABCC }, + + { 0x00000000,0xA6770BB4,0x979F1129,0x31E81A9D,0xF44F2413,0x52382FA7,0x63D0353A,0xC5A73E8E, + 0x33EF4E67,0x959845D3,0xA4705F4E,0x020754FA,0xC7A06A74,0x61D761C0,0x503F7B5D,0xF64870E9, + 0x67DE9CCE,0xC1A9977A,0xF0418DE7,0x56368653,0x9391B8DD,0x35E6B369,0x040EA9F4,0xA279A240, + 0x5431D2A9,0xF246D91D,0xC3AEC380,0x65D9C834,0xA07EF6BA,0x0609FD0E,0x37E1E793,0x9196EC27, + 0xCFBD399C,0x69CA3228,0x582228B5,0xFE552301,0x3BF21D8F,0x9D85163B,0xAC6D0CA6,0x0A1A0712, + 0xFC5277FB,0x5A257C4F,0x6BCD66D2,0xCDBA6D66,0x081D53E8,0xAE6A585C,0x9F8242C1,0x39F54975, + 0xA863A552,0x0E14AEE6,0x3FFCB47B,0x998BBFCF,0x5C2C8141,0xFA5B8AF5,0xCBB39068,0x6DC49BDC, + 0x9B8CEB35,0x3DFBE081,0x0C13FA1C,0xAA64F1A8,0x6FC3CF26,0xC9B4C492,0xF85CDE0F,0x5E2BD5BB, + 0x440B7579,0xE27C7ECD,0xD3946450,0x75E36FE4,0xB044516A,0x16335ADE,0x27DB4043,0x81AC4BF7, + 0x77E43B1E,0xD19330AA,0xE07B2A37,0x460C2183,0x83AB1F0D,0x25DC14B9,0x14340E24,0xB2430590, + 0x23D5E9B7,0x85A2E203,0xB44AF89E,0x123DF32A,0xD79ACDA4,0x71EDC610,0x4005DC8D,0xE672D739, + 0x103AA7D0,0xB64DAC64,0x87A5B6F9,0x21D2BD4D,0xE47583C3,0x42028877,0x73EA92EA,0xD59D995E, + 0x8BB64CE5,0x2DC14751,0x1C295DCC,0xBA5E5678,0x7FF968F6,0xD98E6342,0xE86679DF,0x4E11726B, + 0xB8590282,0x1E2E0936,0x2FC613AB,0x89B1181F,0x4C162691,0xEA612D25,0xDB8937B8,0x7DFE3C0C, + 0xEC68D02B,0x4A1FDB9F,0x7BF7C102,0xDD80CAB6,0x1827F438,0xBE50FF8C,0x8FB8E511,0x29CFEEA5, + 0xDF879E4C,0x79F095F8,0x48188F65,0xEE6F84D1,0x2BC8BA5F,0x8DBFB1EB,0xBC57AB76,0x1A20A0C2, + 0x8816EAF2,0x2E61E146,0x1F89FBDB,0xB9FEF06F,0x7C59CEE1,0xDA2EC555,0xEBC6DFC8,0x4DB1D47C, + 0xBBF9A495,0x1D8EAF21,0x2C66B5BC,0x8A11BE08,0x4FB68086,0xE9C18B32,0xD82991AF,0x7E5E9A1B, + 0xEFC8763C,0x49BF7D88,0x78576715,0xDE206CA1,0x1B87522F,0xBDF0599B,0x8C184306,0x2A6F48B2, + 0xDC27385B,0x7A5033EF,0x4BB82972,0xEDCF22C6,0x28681C48,0x8E1F17FC,0xBFF70D61,0x198006D5, + 0x47ABD36E,0xE1DCD8DA,0xD034C247,0x7643C9F3,0xB3E4F77D,0x1593FCC9,0x247BE654,0x820CEDE0, + 0x74449D09,0xD23396BD,0xE3DB8C20,0x45AC8794,0x800BB91A,0x267CB2AE,0x1794A833,0xB1E3A387, + 0x20754FA0,0x86024414,0xB7EA5E89,0x119D553D,0xD43A6BB3,0x724D6007,0x43A57A9A,0xE5D2712E, + 0x139A01C7,0xB5ED0A73,0x840510EE,0x22721B5A,0xE7D525D4,0x41A22E60,0x704A34FD,0xD63D3F49, + 0xCC1D9F8B,0x6A6A943F,0x5B828EA2,0xFDF58516,0x3852BB98,0x9E25B02C,0xAFCDAAB1,0x09BAA105, + 0xFFF2D1EC,0x5985DA58,0x686DC0C5,0xCE1ACB71,0x0BBDF5FF,0xADCAFE4B,0x9C22E4D6,0x3A55EF62, + 0xABC30345,0x0DB408F1,0x3C5C126C,0x9A2B19D8,0x5F8C2756,0xF9FB2CE2,0xC813367F,0x6E643DCB, + 0x982C4D22,0x3E5B4696,0x0FB35C0B,0xA9C457BF,0x6C636931,0xCA146285,0xFBFC7818,0x5D8B73AC, + 0x03A0A617,0xA5D7ADA3,0x943FB73E,0x3248BC8A,0xF7EF8204,0x519889B0,0x6070932D,0xC6079899, + 0x304FE870,0x9638E3C4,0xA7D0F959,0x01A7F2ED,0xC400CC63,0x6277C7D7,0x539FDD4A,0xF5E8D6FE, + 0x647E3AD9,0xC209316D,0xF3E12BF0,0x55962044,0x90311ECA,0x3646157E,0x07AE0FE3,0xA1D90457, + 0x579174BE,0xF1E67F0A,0xC00E6597,0x66796E23,0xA3DE50AD,0x05A95B19,0x34414184,0x92364A30 }, + + { 0x00000000,0xCCAA009E,0x4225077D,0x8E8F07E3,0x844A0EFA,0x48E00E64,0xC66F0987,0x0AC50919, + 0xD3E51BB5,0x1F4F1B2B,0x91C01CC8,0x5D6A1C56,0x57AF154F,0x9B0515D1,0x158A1232,0xD92012AC, + 0x7CBB312B,0xB01131B5,0x3E9E3656,0xF23436C8,0xF8F13FD1,0x345B3F4F,0xBAD438AC,0x767E3832, + 0xAF5E2A9E,0x63F42A00,0xED7B2DE3,0x21D12D7D,0x2B142464,0xE7BE24FA,0x69312319,0xA59B2387, + 0xF9766256,0x35DC62C8,0xBB53652B,0x77F965B5,0x7D3C6CAC,0xB1966C32,0x3F196BD1,0xF3B36B4F, + 0x2A9379E3,0xE639797D,0x68B67E9E,0xA41C7E00,0xAED97719,0x62737787,0xECFC7064,0x205670FA, + 0x85CD537D,0x496753E3,0xC7E85400,0x0B42549E,0x01875D87,0xCD2D5D19,0x43A25AFA,0x8F085A64, + 0x562848C8,0x9A824856,0x140D4FB5,0xD8A74F2B,0xD2624632,0x1EC846AC,0x9047414F,0x5CED41D1, + 0x299DC2ED,0xE537C273,0x6BB8C590,0xA712C50E,0xADD7CC17,0x617DCC89,0xEFF2CB6A,0x2358CBF4, + 0xFA78D958,0x36D2D9C6,0xB85DDE25,0x74F7DEBB,0x7E32D7A2,0xB298D73C,0x3C17D0DF,0xF0BDD041, + 0x5526F3C6,0x998CF358,0x1703F4BB,0xDBA9F425,0xD16CFD3C,0x1DC6FDA2,0x9349FA41,0x5FE3FADF, + 0x86C3E873,0x4A69E8ED,0xC4E6EF0E,0x084CEF90,0x0289E689,0xCE23E617,0x40ACE1F4,0x8C06E16A, + 0xD0EBA0BB,0x1C41A025,0x92CEA7C6,0x5E64A758,0x54A1AE41,0x980BAEDF,0x1684A93C,0xDA2EA9A2, + 0x030EBB0E,0xCFA4BB90,0x412BBC73,0x8D81BCED,0x8744B5F4,0x4BEEB56A,0xC561B289,0x09CBB217, + 0xAC509190,0x60FA910E,0xEE7596ED,0x22DF9673,0x281A9F6A,0xE4B09FF4,0x6A3F9817,0xA6959889, + 0x7FB58A25,0xB31F8ABB,0x3D908D58,0xF13A8DC6,0xFBFF84DF,0x37558441,0xB9DA83A2,0x7570833C, + 0x533B85DA,0x9F918544,0x111E82A7,0xDDB48239,0xD7718B20,0x1BDB8BBE,0x95548C5D,0x59FE8CC3, + 0x80DE9E6F,0x4C749EF1,0xC2FB9912,0x0E51998C,0x04949095,0xC83E900B,0x46B197E8,0x8A1B9776, + 0x2F80B4F1,0xE32AB46F,0x6DA5B38C,0xA10FB312,0xABCABA0B,0x6760BA95,0xE9EFBD76,0x2545BDE8, + 0xFC65AF44,0x30CFAFDA,0xBE40A839,0x72EAA8A7,0x782FA1BE,0xB485A120,0x3A0AA6C3,0xF6A0A65D, + 0xAA4DE78C,0x66E7E712,0xE868E0F1,0x24C2E06F,0x2E07E976,0xE2ADE9E8,0x6C22EE0B,0xA088EE95, + 0x79A8FC39,0xB502FCA7,0x3B8DFB44,0xF727FBDA,0xFDE2F2C3,0x3148F25D,0xBFC7F5BE,0x736DF520, + 0xD6F6D6A7,0x1A5CD639,0x94D3D1DA,0x5879D144,0x52BCD85D,0x9E16D8C3,0x1099DF20,0xDC33DFBE, + 0x0513CD12,0xC9B9CD8C,0x4736CA6F,0x8B9CCAF1,0x8159C3E8,0x4DF3C376,0xC37CC495,0x0FD6C40B, + 0x7AA64737,0xB60C47A9,0x3883404A,0xF42940D4,0xFEEC49CD,0x32464953,0xBCC94EB0,0x70634E2E, + 0xA9435C82,0x65E95C1C,0xEB665BFF,0x27CC5B61,0x2D095278,0xE1A352E6,0x6F2C5505,0xA386559B, + 0x061D761C,0xCAB77682,0x44387161,0x889271FF,0x825778E6,0x4EFD7878,0xC0727F9B,0x0CD87F05, + 0xD5F86DA9,0x19526D37,0x97DD6AD4,0x5B776A4A,0x51B26353,0x9D1863CD,0x1397642E,0xDF3D64B0, + 0x83D02561,0x4F7A25FF,0xC1F5221C,0x0D5F2282,0x079A2B9B,0xCB302B05,0x45BF2CE6,0x89152C78, + 0x50353ED4,0x9C9F3E4A,0x121039A9,0xDEBA3937,0xD47F302E,0x18D530B0,0x965A3753,0x5AF037CD, + 0xFF6B144A,0x33C114D4,0xBD4E1337,0x71E413A9,0x7B211AB0,0xB78B1A2E,0x39041DCD,0xF5AE1D53, + 0x2C8E0FFF,0xE0240F61,0x6EAB0882,0xA201081C,0xA8C40105,0x646E019B,0xEAE10678,0x264B06E6 } + }; + + inline uint32_t swap(uint32_t x) + { +#if defined(__GNUC__) || defined(__clang__) + return __builtin_bswap32(x); +#endif +#ifdef MSC_VER + return _byteswap_ulong(x); +#endif + + return (x >> 24) | + ((x >> 8) & 0x0000FF00) | + ((x << 8) & 0x00FF0000) | + (x << 24); + } +} + + +/// add arbitrary number of bytes +void CRC32::add(const void* data, size_t numBytes) +{ + uint32_t* current = (uint32_t*) data; + uint32_t crc = ~m_hash; + + // process eight bytes at once + while (numBytes >= 8) + { +#if defined(__BYTE_ORDER) && (__BYTE_ORDER != 0) && (__BYTE_ORDER == __BIG_ENDIAN) + uint32_t one = *current++ ^ swap(crc); + uint32_t two = *current++; + crc = crc32Lookup[7][ one>>24 ] ^ + crc32Lookup[6][(one>>16) & 0xFF] ^ + crc32Lookup[5][(one>> 8) & 0xFF] ^ + crc32Lookup[4][ one & 0xFF] ^ + crc32Lookup[3][ two>>24 ] ^ + crc32Lookup[2][(two>>16) & 0xFF] ^ + crc32Lookup[1][(two>> 8) & 0xFF] ^ + crc32Lookup[0][ two & 0xFF]; +#else + uint32_t one = *current++ ^ crc; + uint32_t two = *current++; + crc = crc32Lookup[7][ one & 0xFF] ^ + crc32Lookup[6][(one>> 8) & 0xFF] ^ + crc32Lookup[5][(one>>16) & 0xFF] ^ + crc32Lookup[4][ one>>24 ] ^ + crc32Lookup[3][ two & 0xFF] ^ + crc32Lookup[2][(two>> 8) & 0xFF] ^ + crc32Lookup[1][(two>>16) & 0xFF] ^ + crc32Lookup[0][ two>>24 ]; +#endif + numBytes -= 8; + } + + unsigned char* currentChar = (unsigned char*) current; + // remaining 1 to 7 bytes (standard CRC table-based algorithm) + while (numBytes--) + crc = (crc >> 8) ^ crc32Lookup[0][(crc & 0xFF) ^ *currentChar++]; + + m_hash = ~crc; +} + + +/// return latest hash as 8 hex characters +std::string CRC32::getHash() +{ + // convert hash to string + static const char dec2hex[16+1] = "0123456789abcdef"; + + char hashBuffer[8+1]; + + hashBuffer[0] = dec2hex[ m_hash >> 28 ]; + hashBuffer[1] = dec2hex[(m_hash >> 24) & 15]; + hashBuffer[2] = dec2hex[(m_hash >> 20) & 15]; + hashBuffer[3] = dec2hex[(m_hash >> 16) & 15]; + hashBuffer[4] = dec2hex[(m_hash >> 12) & 15]; + hashBuffer[5] = dec2hex[(m_hash >> 8) & 15]; + hashBuffer[6] = dec2hex[(m_hash >> 4) & 15]; + hashBuffer[7] = dec2hex[ m_hash & 15]; + // zero-terminated string + hashBuffer[8] = 0; + + // convert to std::string + return hashBuffer; +} + + +/// return latest hash as bytes +void CRC32::getHash(unsigned char buffer[CRC32::HashBytes]) +{ + buffer[0] = (m_hash >> 24) & 0xFF; + buffer[1] = (m_hash >> 16) & 0xFF; + buffer[2] = (m_hash >> 8) & 0xFF; + buffer[3] = m_hash & 0xFF; +} + + +/// compute CRC32 of a memory block +std::string CRC32::operator()(const void* data, size_t numBytes) +{ + reset(); + add(data, numBytes); + return getHash(); +} + + +/// compute CRC32 of a string, excluding final zero +std::string CRC32::operator()(const std::string& text) +{ + reset(); + add(text.c_str(), text.size()); + return getHash(); +} diff -pruN 1.4-2/3rd-party/hash-library/crc32.h 2.5-1/3rd-party/hash-library/crc32.h --- 1.4-2/3rd-party/hash-library/crc32.h 1970-01-01 00:00:00.000000000 +0000 +++ 2.5-1/3rd-party/hash-library/crc32.h 2022-06-15 07:33:00.000000000 +0000 @@ -0,0 +1,69 @@ +// ////////////////////////////////////////////////////////// +// crc32.h +// Copyright (c) 2014,2015 Stephan Brumme. All rights reserved. +// see http://create.stephan-brumme.com/disclaimer.html +// + +#pragma once + +//#include "hash.h" +#include + +// define fixed size integer types +#ifdef _MSC_VER +// Windows +typedef unsigned __int8 uint8_t; +typedef unsigned __int32 uint32_t; +#else +// GCC +#include +#endif + + +/// compute CRC32 hash, based on Intel's Slicing-by-8 algorithm +/** Usage: + CRC32 crc32; + std::string myHash = crc32("Hello World"); // std::string + std::string myHash2 = crc32("How are you", 11); // arbitrary data, 11 bytes + + // or in a streaming fashion: + + CRC32 crc32; + while (more data available) + crc32.add(pointer to fresh data, number of new bytes); + std::string myHash3 = crc32.getHash(); + + Note: + You can find code for the faster Slicing-by-16 algorithm on my website, too: + http://create.stephan-brumme.com/crc32/ + Its unrolled version is about twice as fast but its look-up table doubled in size as well. + */ +class CRC32 //: public Hash +{ +public: + /// hash is 4 bytes long + enum { HashBytes = 4 }; + + /// same as reset() + CRC32(); + + /// compute CRC32 of a memory block + std::string operator()(const void* data, size_t numBytes); + /// compute CRC32 of a string, excluding final zero + std::string operator()(const std::string& text); + + /// add arbitrary number of bytes + void add(const void* data, size_t numBytes); + + /// return latest hash as 8 hex characters + std::string getHash(); + /// return latest hash as bytes + void getHash(unsigned char buffer[HashBytes]); + + /// restart + void reset(); + +private: + /// hash + uint32_t m_hash; +}; diff -pruN 1.4-2/3rd-party/hash-library/hash.h 2.5-1/3rd-party/hash-library/hash.h --- 1.4-2/3rd-party/hash-library/hash.h 1970-01-01 00:00:00.000000000 +0000 +++ 2.5-1/3rd-party/hash-library/hash.h 2022-06-15 07:33:00.000000000 +0000 @@ -0,0 +1,28 @@ +// ////////////////////////////////////////////////////////// +// hash.h +// Copyright (c) 2014,2015 Stephan Brumme. All rights reserved. +// see http://create.stephan-brumme.com/disclaimer.html +// + +#pragma once + +#include + +/// abstract base class +class Hash +{ +public: + /// compute hash of a memory block + virtual std::string operator()(const void* data, size_t numBytes) = 0; + /// compute hash of a string, excluding final zero + virtual std::string operator()(const std::string& text) = 0; + + /// add arbitrary number of bytes + virtual void add(const void* data, size_t numBytes) = 0; + + /// return latest hash as hex characters + virtual std::string getHash() = 0; + + /// restart + virtual void reset() = 0; +}; diff -pruN 1.4-2/3rd-party/hash-library/hmac.h 2.5-1/3rd-party/hash-library/hmac.h --- 1.4-2/3rd-party/hash-library/hmac.h 1970-01-01 00:00:00.000000000 +0000 +++ 2.5-1/3rd-party/hash-library/hmac.h 2022-06-15 07:33:00.000000000 +0000 @@ -0,0 +1,83 @@ +// ////////////////////////////////////////////////////////// +// hmac.h +// Copyright (c) 2015 Stephan Brumme. All rights reserved. +// see http://create.stephan-brumme.com/disclaimer.html +// + +#pragma once + +// based on http://tools.ietf.org/html/rfc2104 +// see also http://en.wikipedia.org/wiki/Hash-based_message_authentication_code + +/** Usage: + std::string msg = "The quick brown fox jumps over the lazy dog"; + std::string key = "key"; + std::string md5hmac = hmac< MD5 >(msg, key); + std::string sha1hmac = hmac< SHA1 >(msg, key); + std::string sha2hmac = hmac(msg, key); + + Note: + To keep my code simple, HMAC computation currently needs the whole message at once. + This is in contrast to the hashes MD5, SHA1, etc. where an add() method is available + for incremental computation. + You can use any hash for HMAC as long as it provides: + - constant HashMethod::BlockSize (typically 64) + - constant HashMethod::HashBytes (length of hash in bytes, e.g. 20 for SHA1) + - HashMethod::add(buffer, bufferSize) + - HashMethod::getHash(unsigned char buffer[HashMethod::BlockSize]) + */ + +#include +#include // memcpy + +/// compute HMAC hash of data and key using MD5, SHA1 or SHA256 +template +std::string hmac(const void* data, size_t numDataBytes, const void* key, size_t numKeyBytes) +{ + // initialize key with zeros + unsigned char usedKey[HashMethod::BlockSize] = {0}; + + // adjust length of key: must contain exactly blockSize bytes + if (numKeyBytes <= HashMethod::BlockSize) + { + // copy key + memcpy(usedKey, key, numKeyBytes); + } + else + { + // shorten key: usedKey = hashed(key) + HashMethod keyHasher; + keyHasher.add(key, numKeyBytes); + keyHasher.getHash(usedKey); + } + + // create initial XOR padding + for (size_t i = 0; i < HashMethod::BlockSize; i++) + usedKey[i] ^= 0x36; + + // inside = hash((usedKey ^ 0x36) + data) + unsigned char inside[HashMethod::HashBytes]; + HashMethod insideHasher; + insideHasher.add(usedKey, HashMethod::BlockSize); + insideHasher.add(data, numDataBytes); + insideHasher.getHash(inside); + + // undo usedKey's previous 0x36 XORing and apply a XOR by 0x5C + for (size_t i = 0; i < HashMethod::BlockSize; i++) + usedKey[i] ^= 0x5C ^ 0x36; + + // hash((usedKey ^ 0x5C) + hash((usedKey ^ 0x36) + data)) + HashMethod finalHasher; + finalHasher.add(usedKey, HashMethod::BlockSize); + finalHasher.add(inside, HashMethod::HashBytes); + + return finalHasher.getHash(); +} + + +/// convenience function for std::string +template +std::string hmac(const std::string& data, const std::string& key) +{ + return hmac(data.c_str(), data.size(), key.c_str(), key.size()); +} diff -pruN 1.4-2/3rd-party/hash-library/keccak.cpp 2.5-1/3rd-party/hash-library/keccak.cpp --- 1.4-2/3rd-party/hash-library/keccak.cpp 1970-01-01 00:00:00.000000000 +0000 +++ 2.5-1/3rd-party/hash-library/keccak.cpp 2022-06-15 07:33:00.000000000 +0000 @@ -0,0 +1,298 @@ +// ////////////////////////////////////////////////////////// +// keccak.cpp +// Copyright (c) 2014,2015 Stephan Brumme. All rights reserved. +// see http://create.stephan-brumme.com/disclaimer.html +// + +#include "keccak.h" + +// big endian architectures need #define __BYTE_ORDER __BIG_ENDIAN +#ifndef _MSC_VER +#include +#endif + + +/// same as reset() +Keccak::Keccak(Bits bits) +: m_blockSize(200 - 2 * (bits / 8)), + m_bits(bits) +{ + reset(); +} + + +/// restart +void Keccak::reset() +{ + for (size_t i = 0; i < StateSize; i++) + m_hash[i] = 0; + + m_numBytes = 0; + m_bufferSize = 0; +} + + +/// constants and local helper functions +namespace +{ + const unsigned int KeccakRounds = 24; + const uint64_t XorMasks[KeccakRounds] = + { + 0x0000000000000001ULL, 0x0000000000008082ULL, 0x800000000000808aULL, + 0x8000000080008000ULL, 0x000000000000808bULL, 0x0000000080000001ULL, + 0x8000000080008081ULL, 0x8000000000008009ULL, 0x000000000000008aULL, + 0x0000000000000088ULL, 0x0000000080008009ULL, 0x000000008000000aULL, + 0x000000008000808bULL, 0x800000000000008bULL, 0x8000000000008089ULL, + 0x8000000000008003ULL, 0x8000000000008002ULL, 0x8000000000000080ULL, + 0x000000000000800aULL, 0x800000008000000aULL, 0x8000000080008081ULL, + 0x8000000000008080ULL, 0x0000000080000001ULL, 0x8000000080008008ULL + }; + + /// rotate left and wrap around to the right + inline uint64_t rotateLeft(uint64_t x, uint8_t numBits) + { + return (x << numBits) | (x >> (64 - numBits)); + } + + /// convert litte vs big endian + inline uint64_t swap(uint64_t x) + { +#if defined(__GNUC__) || defined(__clang__) + return __builtin_bswap64(x); +#endif +#ifdef _MSC_VER + return _byteswap_uint64(x); +#endif + + return (x >> 56) | + ((x >> 40) & 0x000000000000FF00ULL) | + ((x >> 24) & 0x0000000000FF0000ULL) | + ((x >> 8) & 0x00000000FF000000ULL) | + ((x << 8) & 0x000000FF00000000ULL) | + ((x << 24) & 0x0000FF0000000000ULL) | + ((x << 40) & 0x00FF000000000000ULL) | + (x << 56); + } + + + /// return x % 5 for 0 <= x <= 9 + unsigned int mod5(unsigned int x) + { + if (x < 5) + return x; + + return x - 5; + } +} + + +/// process a full block +void Keccak::processBlock(const void* data) +{ +#if defined(__BYTE_ORDER) && (__BYTE_ORDER != 0) && (__BYTE_ORDER == __BIG_ENDIAN) +#define LITTLEENDIAN(x) swap(x) +#else +#define LITTLEENDIAN(x) (x) +#endif + + const uint64_t* data64 = (const uint64_t*) data; + // mix data into state + for (unsigned int i = 0; i < m_blockSize / 8; i++) + m_hash[i] ^= LITTLEENDIAN(data64[i]); + + // re-compute state + for (unsigned int round = 0; round < KeccakRounds; round++) + { + // Theta + uint64_t coefficients[5]; + for (unsigned int i = 0; i < 5; i++) + coefficients[i] = m_hash[i] ^ m_hash[i + 5] ^ m_hash[i + 10] ^ m_hash[i + 15] ^ m_hash[i + 20]; + + for (unsigned int i = 0; i < 5; i++) + { + uint64_t one = coefficients[mod5(i + 4)] ^ rotateLeft(coefficients[mod5(i + 1)], 1); + m_hash[i ] ^= one; + m_hash[i + 5] ^= one; + m_hash[i + 10] ^= one; + m_hash[i + 15] ^= one; + m_hash[i + 20] ^= one; + } + + // temporary + uint64_t one; + + // Rho Pi + uint64_t last = m_hash[1]; + one = m_hash[10]; m_hash[10] = rotateLeft(last, 1); last = one; + one = m_hash[ 7]; m_hash[ 7] = rotateLeft(last, 3); last = one; + one = m_hash[11]; m_hash[11] = rotateLeft(last, 6); last = one; + one = m_hash[17]; m_hash[17] = rotateLeft(last, 10); last = one; + one = m_hash[18]; m_hash[18] = rotateLeft(last, 15); last = one; + one = m_hash[ 3]; m_hash[ 3] = rotateLeft(last, 21); last = one; + one = m_hash[ 5]; m_hash[ 5] = rotateLeft(last, 28); last = one; + one = m_hash[16]; m_hash[16] = rotateLeft(last, 36); last = one; + one = m_hash[ 8]; m_hash[ 8] = rotateLeft(last, 45); last = one; + one = m_hash[21]; m_hash[21] = rotateLeft(last, 55); last = one; + one = m_hash[24]; m_hash[24] = rotateLeft(last, 2); last = one; + one = m_hash[ 4]; m_hash[ 4] = rotateLeft(last, 14); last = one; + one = m_hash[15]; m_hash[15] = rotateLeft(last, 27); last = one; + one = m_hash[23]; m_hash[23] = rotateLeft(last, 41); last = one; + one = m_hash[19]; m_hash[19] = rotateLeft(last, 56); last = one; + one = m_hash[13]; m_hash[13] = rotateLeft(last, 8); last = one; + one = m_hash[12]; m_hash[12] = rotateLeft(last, 25); last = one; + one = m_hash[ 2]; m_hash[ 2] = rotateLeft(last, 43); last = one; + one = m_hash[20]; m_hash[20] = rotateLeft(last, 62); last = one; + one = m_hash[14]; m_hash[14] = rotateLeft(last, 18); last = one; + one = m_hash[22]; m_hash[22] = rotateLeft(last, 39); last = one; + one = m_hash[ 9]; m_hash[ 9] = rotateLeft(last, 61); last = one; + one = m_hash[ 6]; m_hash[ 6] = rotateLeft(last, 20); last = one; + m_hash[ 1] = rotateLeft(last, 44); + + // Chi + for (unsigned int j = 0; j < StateSize; j += 5) + { + // temporaries + uint64_t one = m_hash[j]; + uint64_t two = m_hash[j + 1]; + + m_hash[j] ^= m_hash[j + 2] & ~two; + m_hash[j + 1] ^= m_hash[j + 3] & ~m_hash[j + 2]; + m_hash[j + 2] ^= m_hash[j + 4] & ~m_hash[j + 3]; + m_hash[j + 3] ^= one & ~m_hash[j + 4]; + m_hash[j + 4] ^= two & ~one; + } + + // Iota + m_hash[0] ^= XorMasks[round]; + } +} + + +/// add arbitrary number of bytes +void Keccak::add(const void* data, size_t numBytes) +{ + const uint8_t* current = (const uint8_t*) data; + + if (m_bufferSize > 0) + { + while (numBytes > 0 && m_bufferSize < m_blockSize) + { + m_buffer[m_bufferSize++] = *current++; + numBytes--; + } + } + + // full buffer + if (m_bufferSize == m_blockSize) + { + processBlock((void*)m_buffer); + m_numBytes += m_blockSize; + m_bufferSize = 0; + } + + // no more data ? + if (numBytes == 0) + return; + + // process full blocks + while (numBytes >= m_blockSize) + { + processBlock(current); + current += m_blockSize; + m_numBytes += m_blockSize; + numBytes -= m_blockSize; + } + + // keep remaining bytes in buffer + while (numBytes > 0) + { + m_buffer[m_bufferSize++] = *current++; + numBytes--; + } +} + + +/// process everything left in the internal buffer +void Keccak::processBuffer() +{ + unsigned int blockSize = 200 - 2 * (m_bits / 8); + + // add padding + size_t offset = m_bufferSize; + // add a "1" byte + m_buffer[offset++] = 1; + // fill with zeros + while (offset < blockSize) + m_buffer[offset++] = 0; + + // and add a single set bit + m_buffer[blockSize - 1] |= 0x80; + + processBlock(m_buffer); +} + + +/// return latest hash as 16 hex characters +std::string Keccak::getHash() +{ + // save hash state + uint64_t oldHash[StateSize]; + for (unsigned int i = 0; i < StateSize; i++) + oldHash[i] = m_hash[i]; + + // process remaining bytes + processBuffer(); + + // convert hash to string + static const char dec2hex[16 + 1] = "0123456789abcdef"; + + // number of significant elements in hash (uint64_t) + unsigned int hashLength = m_bits / 64; + + std::string result; + for (unsigned int i = 0; i < hashLength; i++) + for (unsigned int j = 0; j < 8; j++) // 64 bits => 8 bytes + { + // convert a byte to hex + unsigned char oneByte = (unsigned char) (m_hash[i] >> (8 * j)); + result += dec2hex[oneByte >> 4]; + result += dec2hex[oneByte & 15]; + } + + // Keccak224's last entry in m_hash provides only 32 bits instead of 64 bits + unsigned int remainder = m_bits - hashLength * 64; + unsigned int processed = 0; + while (processed < remainder) + { + // convert a byte to hex + unsigned char oneByte = (unsigned char) (m_hash[hashLength] >> processed); + result += dec2hex[oneByte >> 4]; + result += dec2hex[oneByte & 15]; + + processed += 8; + } + + // restore state + for (unsigned int i = 0; i < StateSize; i++) + m_hash[i] = oldHash[i]; + + return result; +} + + +/// compute Keccak hash of a memory block +std::string Keccak::operator()(const void* data, size_t numBytes) +{ + reset(); + add(data, numBytes); + return getHash(); +} + + +/// compute Keccak hash of a string, excluding final zero +std::string Keccak::operator()(const std::string& text) +{ + reset(); + add(text.c_str(), text.size()); + return getHash(); +} diff -pruN 1.4-2/3rd-party/hash-library/keccak.h 2.5-1/3rd-party/hash-library/keccak.h --- 1.4-2/3rd-party/hash-library/keccak.h 1970-01-01 00:00:00.000000000 +0000 +++ 2.5-1/3rd-party/hash-library/keccak.h 2022-06-15 07:33:00.000000000 +0000 @@ -0,0 +1,81 @@ +// ////////////////////////////////////////////////////////// +// keccak.h +// Copyright (c) 2014,2015 Stephan Brumme. All rights reserved. +// see http://create.stephan-brumme.com/disclaimer.html +// + +#pragma once + +//#include "hash.h" +#include + +// define fixed size integer types +#ifdef _MSC_VER +// Windows +typedef unsigned __int8 uint8_t; +typedef unsigned __int64 uint64_t; +#else +// GCC +#include +#endif + + +/// compute Keccak hash (designated SHA3) +/** Usage: + Keccak keccak; + std::string myHash = keccak("Hello World"); // std::string + std::string myHash2 = keccak("How are you", 11); // arbitrary data, 11 bytes + + // or in a streaming fashion: + + Keccak keccak; + while (more data available) + keccak.add(pointer to fresh data, number of new bytes); + std::string myHash3 = keccak.getHash(); + */ +class Keccak //: public Hash +{ +public: + /// algorithm variants + enum Bits { Keccak224 = 224, Keccak256 = 256, Keccak384 = 384, Keccak512 = 512 }; + + /// same as reset() + explicit Keccak(Bits bits = Keccak256); + + /// compute hash of a memory block + std::string operator()(const void* data, size_t numBytes); + /// compute hash of a string, excluding final zero + std::string operator()(const std::string& text); + + /// add arbitrary number of bytes + void add(const void* data, size_t numBytes); + + /// return latest hash as hex characters + std::string getHash(); + + /// restart + void reset(); + +private: + /// process a full block + void processBlock(const void* data); + /// process everything left in the internal buffer + void processBuffer(); + + /// 1600 bits, stored as 25x64 bit, BlockSize is no more than 1152 bits (Keccak224) + enum { StateSize = 1600 / (8 * 8), + MaxBlockSize = 200 - 2 * (224 / 8) }; + + /// hash + uint64_t m_hash[StateSize]; + /// size of processed data in bytes + uint64_t m_numBytes; + /// block size (less or equal to MaxBlockSize) + size_t m_blockSize; + /// valid bytes in m_buffer + size_t m_bufferSize; + /// bytes not processed yet + uint8_t m_buffer[MaxBlockSize]; + /// variant + Bits m_bits; +}; diff -pruN 1.4-2/3rd-party/hash-library/LICENSE 2.5-1/3rd-party/hash-library/LICENSE --- 1.4-2/3rd-party/hash-library/LICENSE 1970-01-01 00:00:00.000000000 +0000 +++ 2.5-1/3rd-party/hash-library/LICENSE 2022-06-15 07:33:00.000000000 +0000 @@ -0,0 +1,10 @@ +zlib License + +Copyright (c) 2014,2015 Stephan Brumme + +This software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages arising from the use of this software. +Permission is granted to anyone to use this software for any purpose, including commercial applications, and to alter it and redistribute it freely, subject to the following restrictions: +1. The origin of this software must not be misrepresented; you must not claim that you wrote the original software. + If you use this software in a product, an acknowledgment in the product documentation would be appreciated but is not required. +2. Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software. +3. This notice may not be removed or altered from any source distribution. diff -pruN 1.4-2/3rd-party/hash-library/md5.cpp 2.5-1/3rd-party/hash-library/md5.cpp --- 1.4-2/3rd-party/hash-library/md5.cpp 1970-01-01 00:00:00.000000000 +0000 +++ 2.5-1/3rd-party/hash-library/md5.cpp 2022-06-15 07:33:00.000000000 +0000 @@ -0,0 +1,380 @@ +// ////////////////////////////////////////////////////////// +// md5.cpp +// Copyright (c) 2014,2015 Stephan Brumme. All rights reserved. +// see http://create.stephan-brumme.com/disclaimer.html +// + +#include "md5.h" + +#ifndef _MSC_VER +#include +#endif + + +/// same as reset() +MD5::MD5() +{ + reset(); +} + + +/// restart +void MD5::reset() +{ + m_numBytes = 0; + m_bufferSize = 0; + + // according to RFC 1321 + m_hash[0] = 0x67452301; + m_hash[1] = 0xefcdab89; + m_hash[2] = 0x98badcfe; + m_hash[3] = 0x10325476; +} + + +namespace +{ + // mix functions for processBlock() + inline uint32_t f1(uint32_t b, uint32_t c, uint32_t d) + { + return d ^ (b & (c ^ d)); // original: f = (b & c) | ((~b) & d); + } + + inline uint32_t f2(uint32_t b, uint32_t c, uint32_t d) + { + return c ^ (d & (b ^ c)); // original: f = (b & d) | (c & (~d)); + } + + inline uint32_t f3(uint32_t b, uint32_t c, uint32_t d) + { + return b ^ c ^ d; + } + + inline uint32_t f4(uint32_t b, uint32_t c, uint32_t d) + { + return c ^ (b | ~d); + } + + inline uint32_t rotate(uint32_t a, uint32_t c) + { + return (a << c) | (a >> (32 - c)); + } + +#if defined(__BYTE_ORDER) && (__BYTE_ORDER != 0) && (__BYTE_ORDER == __BIG_ENDIAN) + inline uint32_t swap(uint32_t x) + { +#if defined(__GNUC__) || defined(__clang__) + return __builtin_bswap32(x); +#endif +#ifdef MSC_VER + return _byteswap_ulong(x); +#endif + + return (x >> 24) | + ((x >> 8) & 0x0000FF00) | + ((x << 8) & 0x00FF0000) | + (x << 24); + } +#endif +} + + +/// process 64 bytes +void MD5::processBlock(const void* data) +{ + // get last hash + uint32_t a = m_hash[0]; + uint32_t b = m_hash[1]; + uint32_t c = m_hash[2]; + uint32_t d = m_hash[3]; + + // data represented as 16x 32-bit words + const uint32_t* words = (uint32_t*) data; + + // computations are little endian, swap data if necessary +#if defined(__BYTE_ORDER) && (__BYTE_ORDER != 0) && (__BYTE_ORDER == __BIG_ENDIAN) +#define LITTLEENDIAN(x) swap(x) +#else +#define LITTLEENDIAN(x) (x) +#endif + + // first round + uint32_t word0 = LITTLEENDIAN(words[ 0]); + a = rotate(a + f1(b,c,d) + word0 + 0xd76aa478, 7) + b; + uint32_t word1 = LITTLEENDIAN(words[ 1]); + d = rotate(d + f1(a,b,c) + word1 + 0xe8c7b756, 12) + a; + uint32_t word2 = LITTLEENDIAN(words[ 2]); + c = rotate(c + f1(d,a,b) + word2 + 0x242070db, 17) + d; + uint32_t word3 = LITTLEENDIAN(words[ 3]); + b = rotate(b + f1(c,d,a) + word3 + 0xc1bdceee, 22) + c; + + uint32_t word4 = LITTLEENDIAN(words[ 4]); + a = rotate(a + f1(b,c,d) + word4 + 0xf57c0faf, 7) + b; + uint32_t word5 = LITTLEENDIAN(words[ 5]); + d = rotate(d + f1(a,b,c) + word5 + 0x4787c62a, 12) + a; + uint32_t word6 = LITTLEENDIAN(words[ 6]); + c = rotate(c + f1(d,a,b) + word6 + 0xa8304613, 17) + d; + uint32_t word7 = LITTLEENDIAN(words[ 7]); + b = rotate(b + f1(c,d,a) + word7 + 0xfd469501, 22) + c; + + uint32_t word8 = LITTLEENDIAN(words[ 8]); + a = rotate(a + f1(b,c,d) + word8 + 0x698098d8, 7) + b; + uint32_t word9 = LITTLEENDIAN(words[ 9]); + d = rotate(d + f1(a,b,c) + word9 + 0x8b44f7af, 12) + a; + uint32_t word10 = LITTLEENDIAN(words[10]); + c = rotate(c + f1(d,a,b) + word10 + 0xffff5bb1, 17) + d; + uint32_t word11 = LITTLEENDIAN(words[11]); + b = rotate(b + f1(c,d,a) + word11 + 0x895cd7be, 22) + c; + + uint32_t word12 = LITTLEENDIAN(words[12]); + a = rotate(a + f1(b,c,d) + word12 + 0x6b901122, 7) + b; + uint32_t word13 = LITTLEENDIAN(words[13]); + d = rotate(d + f1(a,b,c) + word13 + 0xfd987193, 12) + a; + uint32_t word14 = LITTLEENDIAN(words[14]); + c = rotate(c + f1(d,a,b) + word14 + 0xa679438e, 17) + d; + uint32_t word15 = LITTLEENDIAN(words[15]); + b = rotate(b + f1(c,d,a) + word15 + 0x49b40821, 22) + c; + + // second round + a = rotate(a + f2(b,c,d) + word1 + 0xf61e2562, 5) + b; + d = rotate(d + f2(a,b,c) + word6 + 0xc040b340, 9) + a; + c = rotate(c + f2(d,a,b) + word11 + 0x265e5a51, 14) + d; + b = rotate(b + f2(c,d,a) + word0 + 0xe9b6c7aa, 20) + c; + + a = rotate(a + f2(b,c,d) + word5 + 0xd62f105d, 5) + b; + d = rotate(d + f2(a,b,c) + word10 + 0x02441453, 9) + a; + c = rotate(c + f2(d,a,b) + word15 + 0xd8a1e681, 14) + d; + b = rotate(b + f2(c,d,a) + word4 + 0xe7d3fbc8, 20) + c; + + a = rotate(a + f2(b,c,d) + word9 + 0x21e1cde6, 5) + b; + d = rotate(d + f2(a,b,c) + word14 + 0xc33707d6, 9) + a; + c = rotate(c + f2(d,a,b) + word3 + 0xf4d50d87, 14) + d; + b = rotate(b + f2(c,d,a) + word8 + 0x455a14ed, 20) + c; + + a = rotate(a + f2(b,c,d) + word13 + 0xa9e3e905, 5) + b; + d = rotate(d + f2(a,b,c) + word2 + 0xfcefa3f8, 9) + a; + c = rotate(c + f2(d,a,b) + word7 + 0x676f02d9, 14) + d; + b = rotate(b + f2(c,d,a) + word12 + 0x8d2a4c8a, 20) + c; + + // third round + a = rotate(a + f3(b,c,d) + word5 + 0xfffa3942, 4) + b; + d = rotate(d + f3(a,b,c) + word8 + 0x8771f681, 11) + a; + c = rotate(c + f3(d,a,b) + word11 + 0x6d9d6122, 16) + d; + b = rotate(b + f3(c,d,a) + word14 + 0xfde5380c, 23) + c; + + a = rotate(a + f3(b,c,d) + word1 + 0xa4beea44, 4) + b; + d = rotate(d + f3(a,b,c) + word4 + 0x4bdecfa9, 11) + a; + c = rotate(c + f3(d,a,b) + word7 + 0xf6bb4b60, 16) + d; + b = rotate(b + f3(c,d,a) + word10 + 0xbebfbc70, 23) + c; + + a = rotate(a + f3(b,c,d) + word13 + 0x289b7ec6, 4) + b; + d = rotate(d + f3(a,b,c) + word0 + 0xeaa127fa, 11) + a; + c = rotate(c + f3(d,a,b) + word3 + 0xd4ef3085, 16) + d; + b = rotate(b + f3(c,d,a) + word6 + 0x04881d05, 23) + c; + + a = rotate(a + f3(b,c,d) + word9 + 0xd9d4d039, 4) + b; + d = rotate(d + f3(a,b,c) + word12 + 0xe6db99e5, 11) + a; + c = rotate(c + f3(d,a,b) + word15 + 0x1fa27cf8, 16) + d; + b = rotate(b + f3(c,d,a) + word2 + 0xc4ac5665, 23) + c; + + // fourth round + a = rotate(a + f4(b,c,d) + word0 + 0xf4292244, 6) + b; + d = rotate(d + f4(a,b,c) + word7 + 0x432aff97, 10) + a; + c = rotate(c + f4(d,a,b) + word14 + 0xab9423a7, 15) + d; + b = rotate(b + f4(c,d,a) + word5 + 0xfc93a039, 21) + c; + + a = rotate(a + f4(b,c,d) + word12 + 0x655b59c3, 6) + b; + d = rotate(d + f4(a,b,c) + word3 + 0x8f0ccc92, 10) + a; + c = rotate(c + f4(d,a,b) + word10 + 0xffeff47d, 15) + d; + b = rotate(b + f4(c,d,a) + word1 + 0x85845dd1, 21) + c; + + a = rotate(a + f4(b,c,d) + word8 + 0x6fa87e4f, 6) + b; + d = rotate(d + f4(a,b,c) + word15 + 0xfe2ce6e0, 10) + a; + c = rotate(c + f4(d,a,b) + word6 + 0xa3014314, 15) + d; + b = rotate(b + f4(c,d,a) + word13 + 0x4e0811a1, 21) + c; + + a = rotate(a + f4(b,c,d) + word4 + 0xf7537e82, 6) + b; + d = rotate(d + f4(a,b,c) + word11 + 0xbd3af235, 10) + a; + c = rotate(c + f4(d,a,b) + word2 + 0x2ad7d2bb, 15) + d; + b = rotate(b + f4(c,d,a) + word9 + 0xeb86d391, 21) + c; + + // update hash + m_hash[0] += a; + m_hash[1] += b; + m_hash[2] += c; + m_hash[3] += d; +} + + +/// add arbitrary number of bytes +void MD5::add(const void* data, size_t numBytes) +{ + const uint8_t* current = (const uint8_t*) data; + + if (m_bufferSize > 0) + { + while (numBytes > 0 && m_bufferSize < BlockSize) + { + m_buffer[m_bufferSize++] = *current++; + numBytes--; + } + } + + // full buffer + if (m_bufferSize == BlockSize) + { + processBlock(m_buffer); + m_numBytes += BlockSize; + m_bufferSize = 0; + } + + // no more data ? + if (numBytes == 0) + return; + + // process full blocks + while (numBytes >= BlockSize) + { + processBlock(current); + current += BlockSize; + m_numBytes += BlockSize; + numBytes -= BlockSize; + } + + // keep remaining bytes in buffer + while (numBytes > 0) + { + m_buffer[m_bufferSize++] = *current++; + numBytes--; + } +} + + +/// process final block, less than 64 bytes +void MD5::processBuffer() +{ + // the input bytes are considered as bits strings, where the first bit is the most significant bit of the byte + + // - append "1" bit to message + // - append "0" bits until message length in bit mod 512 is 448 + // - append length as 64 bit integer + + // number of bits + size_t paddedLength = m_bufferSize * 8; + + // plus one bit set to 1 (always appended) + paddedLength++; + + // number of bits must be (numBits % 512) = 448 + size_t lower11Bits = paddedLength & 511; + if (lower11Bits <= 448) + paddedLength += 448 - lower11Bits; + else + paddedLength += 512 + 448 - lower11Bits; + // convert from bits to bytes + paddedLength /= 8; + + // only needed if additional data flows over into a second block + unsigned char extra[BlockSize]; + + // append a "1" bit, 128 => binary 10000000 + if (m_bufferSize < BlockSize) + m_buffer[m_bufferSize] = 128; + else + extra[0] = 128; + + size_t i; + for (i = m_bufferSize + 1; i < BlockSize; i++) + m_buffer[i] = 0; + for (; i < paddedLength; i++) + extra[i - BlockSize] = 0; + + // add message length in bits as 64 bit number + uint64_t msgBits = 8 * (m_numBytes + m_bufferSize); + // find right position + unsigned char* addLength; + if (paddedLength < BlockSize) + addLength = m_buffer + paddedLength; + else + addLength = extra + paddedLength - BlockSize; + + // must be little endian + *addLength++ = msgBits & 0xFF; msgBits >>= 8; + *addLength++ = msgBits & 0xFF; msgBits >>= 8; + *addLength++ = msgBits & 0xFF; msgBits >>= 8; + *addLength++ = msgBits & 0xFF; msgBits >>= 8; + *addLength++ = msgBits & 0xFF; msgBits >>= 8; + *addLength++ = msgBits & 0xFF; msgBits >>= 8; + *addLength++ = msgBits & 0xFF; msgBits >>= 8; + *addLength++ = msgBits & 0xFF; + + // process blocks + processBlock(m_buffer); + // flowed over into a second block ? + if (paddedLength > BlockSize) + processBlock(extra); +} + + +/// return latest hash as 32 hex characters +std::string MD5::getHash() +{ + // compute hash (as raw bytes) + unsigned char rawHash[HashBytes]; + getHash(rawHash); + + // convert to hex string + std::string result; + result.reserve(2 * HashBytes); + for (int i = 0; i < HashBytes; i++) + { + static const char dec2hex[16+1] = "0123456789abcdef"; + result += dec2hex[(rawHash[i] >> 4) & 15]; + result += dec2hex[ rawHash[i] & 15]; + } + + return result; +} + + +/// return latest hash as bytes +void MD5::getHash(unsigned char buffer[MD5::HashBytes]) +{ + // save old hash if buffer is partially filled + uint32_t oldHash[HashValues]; + for (int i = 0; i < HashValues; i++) + oldHash[i] = m_hash[i]; + + // process remaining bytes + processBuffer(); + + unsigned char* current = buffer; + for (int i = 0; i < HashValues; i++) + { + *current++ = m_hash[i] & 0xFF; + *current++ = (m_hash[i] >> 8) & 0xFF; + *current++ = (m_hash[i] >> 16) & 0xFF; + *current++ = (m_hash[i] >> 24) & 0xFF; + + // restore old hash + m_hash[i] = oldHash[i]; + } +} + + +/// compute MD5 of a memory block +std::string MD5::operator()(const void* data, size_t numBytes) +{ + reset(); + add(data, numBytes); + return getHash(); +} + + +/// compute MD5 of a string, excluding final zero +std::string MD5::operator()(const std::string& text) +{ + reset(); + add(text.c_str(), text.size()); + return getHash(); +} diff -pruN 1.4-2/3rd-party/hash-library/md5.h 2.5-1/3rd-party/hash-library/md5.h --- 1.4-2/3rd-party/hash-library/md5.h 1970-01-01 00:00:00.000000000 +0000 +++ 2.5-1/3rd-party/hash-library/md5.h 2022-06-15 07:33:00.000000000 +0000 @@ -0,0 +1,78 @@ +// ////////////////////////////////////////////////////////// +// md5.h +// Copyright (c) 2014 Stephan Brumme. All rights reserved. +// see http://create.stephan-brumme.com/disclaimer.html +// + +#pragma once + +//#include "hash.h" +#include + +// define fixed size integer types +#ifdef _MSC_VER +// Windows +typedef unsigned __int8 uint8_t; +typedef unsigned __int32 uint32_t; +typedef unsigned __int64 uint64_t; +#else +// GCC +#include +#endif + + +/// compute MD5 hash +/** Usage: + MD5 md5; + std::string myHash = md5("Hello World"); // std::string + std::string myHash2 = md5("How are you", 11); // arbitrary data, 11 bytes + + // or in a streaming fashion: + + MD5 md5; + while (more data available) + md5.add(pointer to fresh data, number of new bytes); + std::string myHash3 = md5.getHash(); + */ +class MD5 //: public Hash +{ +public: + /// split into 64 byte blocks (=> 512 bits), hash is 16 bytes long + enum { BlockSize = 512 / 8, HashBytes = 16 }; + + /// same as reset() + MD5(); + + /// compute MD5 of a memory block + std::string operator()(const void* data, size_t numBytes); + /// compute MD5 of a string, excluding final zero + std::string operator()(const std::string& text); + + /// add arbitrary number of bytes + void add(const void* data, size_t numBytes); + + /// return latest hash as 32 hex characters + std::string getHash(); + /// return latest hash as bytes + void getHash(unsigned char buffer[HashBytes]); + + /// restart + void reset(); + +private: + /// process 64 bytes + void processBlock(const void* data); + /// process everything left in the internal buffer + void processBuffer(); + + /// size of processed data in bytes + uint64_t m_numBytes; + /// valid bytes in m_buffer + size_t m_bufferSize; + /// bytes not processed yet + uint8_t m_buffer[BlockSize]; + + enum { HashValues = HashBytes / 4 }; + /// hash, stored as integers + uint32_t m_hash[HashValues]; +}; diff -pruN 1.4-2/3rd-party/hash-library/readme.md 2.5-1/3rd-party/hash-library/readme.md --- 1.4-2/3rd-party/hash-library/readme.md 1970-01-01 00:00:00.000000000 +0000 +++ 2.5-1/3rd-party/hash-library/readme.md 2022-06-15 07:33:00.000000000 +0000 @@ -0,0 +1,51 @@ +# Portable C++ Hashing Library + +This is a mirror of my library hosted at https://create.stephan-brumme.com/hash-library/ + +In a nutshell: + +- computes CRC32, MD5, SHA1 and SHA256 (most common member of the SHA2 functions), Keccak and its SHA3 sibling +- optional HMAC (keyed-hash message authentication code) +- no external dependencies, small code size +- can work chunk-wise (for example when reading streams block-by-block) +- portable: supports Windows and Linux, tested on Little Endian and Big Endian CPUs +- roughly as fast as Linux core hashing functions +- open source, zlib license + +You can find code examples, benchmarks and much more on my website https://create.stephan-brumme.com/hash-library/ + +# How to use + +This example computes SHA256 hashes but the API is more or less identical for all hash algorithms: + +``` cpp +// SHA2 test program +#include "sha256.h" +#include // for std::cout only, not needed for hashing library + +int main(int, char**) +{ + // create a new hashing object + SHA256 sha256; + + // hashing an std::string + std::cout << sha256("Hello World") << std::endl; + // => a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e + + // hashing a buffer of bytes + const char* buffer = "How are you"; + std::cout << sha256(buffer, 11) << std::endl; + // => 9c7d5b046878838da72e40ceb3179580958df544b240869b80d0275cc07209cc + + // or in a streaming fashion (re-use "How are you") + SHA256 sha256stream; + const char* url = "create.stephan-brumme.com"; // 25 bytes + int step = 5; + for (int i = 0; i < 25; i += step) + sha256stream.add(url + i, step); // add five bytes at a time + std::cout << sha256stream.getHash() << std::endl; + // => 82aa771f1183c52f973c798c9243a1c73833ea40961c73e55e12430ec77b69f6 + + return 0; +} +``` diff -pruN 1.4-2/3rd-party/hash-library/sha1.cpp 2.5-1/3rd-party/hash-library/sha1.cpp --- 1.4-2/3rd-party/hash-library/sha1.cpp 1970-01-01 00:00:00.000000000 +0000 +++ 2.5-1/3rd-party/hash-library/sha1.cpp 2022-06-15 07:33:00.000000000 +0000 @@ -0,0 +1,326 @@ +// ////////////////////////////////////////////////////////// +// sha1.cpp +// Copyright (c) 2014,2015 Stephan Brumme. All rights reserved. +// see http://create.stephan-brumme.com/disclaimer.html +// + +#include "sha1.h" + +// big endian architectures need #define __BYTE_ORDER __BIG_ENDIAN +#ifndef _MSC_VER +#include +#endif + + +/// same as reset() +SHA1::SHA1() +{ + reset(); +} + + +/// restart +void SHA1::reset() +{ + m_numBytes = 0; + m_bufferSize = 0; + + // according to RFC 1321 + m_hash[0] = 0x67452301; + m_hash[1] = 0xefcdab89; + m_hash[2] = 0x98badcfe; + m_hash[3] = 0x10325476; + m_hash[4] = 0xc3d2e1f0; +} + + +namespace +{ + // mix functions for processBlock() + inline uint32_t f1(uint32_t b, uint32_t c, uint32_t d) + { + return d ^ (b & (c ^ d)); // original: f = (b & c) | ((~b) & d); + } + + inline uint32_t f2(uint32_t b, uint32_t c, uint32_t d) + { + return b ^ c ^ d; + } + + inline uint32_t f3(uint32_t b, uint32_t c, uint32_t d) + { + return (b & c) | (b & d) | (c & d); + } + + inline uint32_t rotate(uint32_t a, uint32_t c) + { + return (a << c) | (a >> (32 - c)); + } + + inline uint32_t swap(uint32_t x) + { +#if defined(__GNUC__) || defined(__clang__) + return __builtin_bswap32(x); +#endif +#ifdef MSC_VER + return _byteswap_ulong(x); +#endif + + return (x >> 24) | + ((x >> 8) & 0x0000FF00) | + ((x << 8) & 0x00FF0000) | + (x << 24); + } +} + + +/// process 64 bytes +void SHA1::processBlock(const void* data) +{ + // get last hash + uint32_t a = m_hash[0]; + uint32_t b = m_hash[1]; + uint32_t c = m_hash[2]; + uint32_t d = m_hash[3]; + uint32_t e = m_hash[4]; + + // data represented as 16x 32-bit words + const uint32_t* input = (uint32_t*) data; + // convert to big endian + uint32_t words[80]; + for (int i = 0; i < 16; i++) +#if defined(__BYTE_ORDER) && (__BYTE_ORDER != 0) && (__BYTE_ORDER == __BIG_ENDIAN) + words[i] = input[i]; +#else + words[i] = swap(input[i]); +#endif + + // extend to 80 words + for (int i = 16; i < 80; i++) + words[i] = rotate(words[i-3] ^ words[i-8] ^ words[i-14] ^ words[i-16], 1); + + // first round + for (int i = 0; i < 4; i++) + { + int offset = 5*i; + e += rotate(a,5) + f1(b,c,d) + words[offset ] + 0x5a827999; b = rotate(b,30); + d += rotate(e,5) + f1(a,b,c) + words[offset+1] + 0x5a827999; a = rotate(a,30); + c += rotate(d,5) + f1(e,a,b) + words[offset+2] + 0x5a827999; e = rotate(e,30); + b += rotate(c,5) + f1(d,e,a) + words[offset+3] + 0x5a827999; d = rotate(d,30); + a += rotate(b,5) + f1(c,d,e) + words[offset+4] + 0x5a827999; c = rotate(c,30); + } + + // second round + for (int i = 4; i < 8; i++) + { + int offset = 5*i; + e += rotate(a,5) + f2(b,c,d) + words[offset ] + 0x6ed9eba1; b = rotate(b,30); + d += rotate(e,5) + f2(a,b,c) + words[offset+1] + 0x6ed9eba1; a = rotate(a,30); + c += rotate(d,5) + f2(e,a,b) + words[offset+2] + 0x6ed9eba1; e = rotate(e,30); + b += rotate(c,5) + f2(d,e,a) + words[offset+3] + 0x6ed9eba1; d = rotate(d,30); + a += rotate(b,5) + f2(c,d,e) + words[offset+4] + 0x6ed9eba1; c = rotate(c,30); + } + + // third round + for (int i = 8; i < 12; i++) + { + int offset = 5*i; + e += rotate(a,5) + f3(b,c,d) + words[offset ] + 0x8f1bbcdc; b = rotate(b,30); + d += rotate(e,5) + f3(a,b,c) + words[offset+1] + 0x8f1bbcdc; a = rotate(a,30); + c += rotate(d,5) + f3(e,a,b) + words[offset+2] + 0x8f1bbcdc; e = rotate(e,30); + b += rotate(c,5) + f3(d,e,a) + words[offset+3] + 0x8f1bbcdc; d = rotate(d,30); + a += rotate(b,5) + f3(c,d,e) + words[offset+4] + 0x8f1bbcdc; c = rotate(c,30); + } + + // fourth round + for (int i = 12; i < 16; i++) + { + int offset = 5*i; + e += rotate(a,5) + f2(b,c,d) + words[offset ] + 0xca62c1d6; b = rotate(b,30); + d += rotate(e,5) + f2(a,b,c) + words[offset+1] + 0xca62c1d6; a = rotate(a,30); + c += rotate(d,5) + f2(e,a,b) + words[offset+2] + 0xca62c1d6; e = rotate(e,30); + b += rotate(c,5) + f2(d,e,a) + words[offset+3] + 0xca62c1d6; d = rotate(d,30); + a += rotate(b,5) + f2(c,d,e) + words[offset+4] + 0xca62c1d6; c = rotate(c,30); + } + + // update hash + m_hash[0] += a; + m_hash[1] += b; + m_hash[2] += c; + m_hash[3] += d; + m_hash[4] += e; +} + + +/// add arbitrary number of bytes +void SHA1::add(const void* data, size_t numBytes) +{ + const uint8_t* current = (const uint8_t*) data; + + if (m_bufferSize > 0) + { + while (numBytes > 0 && m_bufferSize < BlockSize) + { + m_buffer[m_bufferSize++] = *current++; + numBytes--; + } + } + + // full buffer + if (m_bufferSize == BlockSize) + { + processBlock((void*)m_buffer); + m_numBytes += BlockSize; + m_bufferSize = 0; + } + + // no more data ? + if (numBytes == 0) + return; + + // process full blocks + while (numBytes >= BlockSize) + { + processBlock(current); + current += BlockSize; + m_numBytes += BlockSize; + numBytes -= BlockSize; + } + + // keep remaining bytes in buffer + while (numBytes > 0) + { + m_buffer[m_bufferSize++] = *current++; + numBytes--; + } +} + + +/// process final block, less than 64 bytes +void SHA1::processBuffer() +{ + // the input bytes are considered as bits strings, where the first bit is the most significant bit of the byte + + // - append "1" bit to message + // - append "0" bits until message length in bit mod 512 is 448 + // - append length as 64 bit integer + + // number of bits + size_t paddedLength = m_bufferSize * 8; + + // plus one bit set to 1 (always appended) + paddedLength++; + + // number of bits must be (numBits % 512) = 448 + size_t lower11Bits = paddedLength & 511; + if (lower11Bits <= 448) + paddedLength += 448 - lower11Bits; + else + paddedLength += 512 + 448 - lower11Bits; + // convert from bits to bytes + paddedLength /= 8; + + // only needed if additional data flows over into a second block + unsigned char extra[BlockSize]; + + // append a "1" bit, 128 => binary 10000000 + if (m_bufferSize < BlockSize) + m_buffer[m_bufferSize] = 128; + else + extra[0] = 128; + + size_t i; + for (i = m_bufferSize + 1; i < BlockSize; i++) + m_buffer[i] = 0; + for (; i < paddedLength; i++) + extra[i - BlockSize] = 0; + + // add message length in bits as 64 bit number + uint64_t msgBits = 8 * (m_numBytes + m_bufferSize); + // find right position + unsigned char* addLength; + if (paddedLength < BlockSize) + addLength = m_buffer + paddedLength; + else + addLength = extra + paddedLength - BlockSize; + + // must be big endian + *addLength++ = (unsigned char)((msgBits >> 56) & 0xFF); + *addLength++ = (unsigned char)((msgBits >> 48) & 0xFF); + *addLength++ = (unsigned char)((msgBits >> 40) & 0xFF); + *addLength++ = (unsigned char)((msgBits >> 32) & 0xFF); + *addLength++ = (unsigned char)((msgBits >> 24) & 0xFF); + *addLength++ = (unsigned char)((msgBits >> 16) & 0xFF); + *addLength++ = (unsigned char)((msgBits >> 8) & 0xFF); + *addLength = (unsigned char)( msgBits & 0xFF); + + // process blocks + processBlock(m_buffer); + // flowed over into a second block ? + if (paddedLength > BlockSize) + processBlock(extra); +} + + +/// return latest hash as 40 hex characters +std::string SHA1::getHash() +{ + // compute hash (as raw bytes) + unsigned char rawHash[HashBytes]; + getHash(rawHash); + + // convert to hex string + std::string result; + result.reserve(2 * HashBytes); + for (int i = 0; i < HashBytes; i++) + { + static const char dec2hex[16+1] = "0123456789abcdef"; + result += dec2hex[(rawHash[i] >> 4) & 15]; + result += dec2hex[ rawHash[i] & 15]; + } + + return result; +} + + +/// return latest hash as bytes +void SHA1::getHash(unsigned char buffer[SHA1::HashBytes]) +{ + // save old hash if buffer is partially filled + uint32_t oldHash[HashValues]; + for (int i = 0; i < HashValues; i++) + oldHash[i] = m_hash[i]; + + // process remaining bytes + processBuffer(); + + unsigned char* current = buffer; + for (int i = 0; i < HashValues; i++) + { + *current++ = (m_hash[i] >> 24) & 0xFF; + *current++ = (m_hash[i] >> 16) & 0xFF; + *current++ = (m_hash[i] >> 8) & 0xFF; + *current++ = m_hash[i] & 0xFF; + + // restore old hash + m_hash[i] = oldHash[i]; + } +} + + +/// compute SHA1 of a memory block +std::string SHA1::operator()(const void* data, size_t numBytes) +{ + reset(); + add(data, numBytes); + return getHash(); +} + + +/// compute SHA1 of a string, excluding final zero +std::string SHA1::operator()(const std::string& text) +{ + reset(); + add(text.c_str(), text.size()); + return getHash(); +} diff -pruN 1.4-2/3rd-party/hash-library/sha1.h 2.5-1/3rd-party/hash-library/sha1.h --- 1.4-2/3rd-party/hash-library/sha1.h 1970-01-01 00:00:00.000000000 +0000 +++ 2.5-1/3rd-party/hash-library/sha1.h 2022-06-15 07:33:00.000000000 +0000 @@ -0,0 +1,78 @@ +// ////////////////////////////////////////////////////////// +// sha1.h +// Copyright (c) 2014,2015 Stephan Brumme. All rights reserved. +// see http://create.stephan-brumme.com/disclaimer.html +// + +#pragma once + +//#include "hash.h" +#include + +// define fixed size integer types +#ifdef _MSC_VER +// Windows +typedef unsigned __int8 uint8_t; +typedef unsigned __int32 uint32_t; +typedef unsigned __int64 uint64_t; +#else +// GCC +#include +#endif + + +/// compute SHA1 hash +/** Usage: + SHA1 sha1; + std::string myHash = sha1("Hello World"); // std::string + std::string myHash2 = sha1("How are you", 11); // arbitrary data, 11 bytes + + // or in a streaming fashion: + + SHA1 sha1; + while (more data available) + sha1.add(pointer to fresh data, number of new bytes); + std::string myHash3 = sha1.getHash(); + */ +class SHA1 //: public Hash +{ +public: + /// split into 64 byte blocks (=> 512 bits), hash is 20 bytes long + enum { BlockSize = 512 / 8, HashBytes = 20 }; + + /// same as reset() + SHA1(); + + /// compute SHA1 of a memory block + std::string operator()(const void* data, size_t numBytes); + /// compute SHA1 of a string, excluding final zero + std::string operator()(const std::string& text); + + /// add arbitrary number of bytes + void add(const void* data, size_t numBytes); + + /// return latest hash as 40 hex characters + std::string getHash(); + /// return latest hash as bytes + void getHash(unsigned char buffer[HashBytes]); + + /// restart + void reset(); + +private: + /// process 64 bytes + void processBlock(const void* data); + /// process everything left in the internal buffer + void processBuffer(); + + /// size of processed data in bytes + uint64_t m_numBytes; + /// valid bytes in m_buffer + size_t m_bufferSize; + /// bytes not processed yet + uint8_t m_buffer[BlockSize]; + + enum { HashValues = HashBytes / 4 }; + /// hash, stored as integers + uint32_t m_hash[HashValues]; +}; diff -pruN 1.4-2/3rd-party/hash-library/sha256.cpp 2.5-1/3rd-party/hash-library/sha256.cpp --- 1.4-2/3rd-party/hash-library/sha256.cpp 1970-01-01 00:00:00.000000000 +0000 +++ 2.5-1/3rd-party/hash-library/sha256.cpp 2022-06-15 07:33:00.000000000 +0000 @@ -0,0 +1,428 @@ +// ////////////////////////////////////////////////////////// +// sha256.cpp +// Copyright (c) 2014,2015,2021 Stephan Brumme. All rights reserved. +// see http://create.stephan-brumme.com/disclaimer.html +// + +#include "sha256.h" + +// big endian architectures need #define __BYTE_ORDER __BIG_ENDIAN +#ifndef _MSC_VER +#include +#endif + +//#define SHA2_224_SEED_VECTOR + + +/// same as reset() +SHA256::SHA256() +{ + reset(); +} + + +/// restart +void SHA256::reset() +{ + m_numBytes = 0; + m_bufferSize = 0; + + // according to RFC 1321 + // "These words were obtained by taking the first thirty-two bits of the + // fractional parts of the square roots of the first eight prime numbers" + m_hash[0] = 0x6a09e667; + m_hash[1] = 0xbb67ae85; + m_hash[2] = 0x3c6ef372; + m_hash[3] = 0xa54ff53a; + m_hash[4] = 0x510e527f; + m_hash[5] = 0x9b05688c; + m_hash[6] = 0x1f83d9ab; + m_hash[7] = 0x5be0cd19; + +#ifdef SHA2_224_SEED_VECTOR + // if you want SHA2-224 instead then use these seeds + // and throw away the last 32 bits of getHash + m_hash[0] = 0xc1059ed8; + m_hash[1] = 0x367cd507; + m_hash[2] = 0x3070dd17; + m_hash[3] = 0xf70e5939; + m_hash[4] = 0xffc00b31; + m_hash[5] = 0x68581511; + m_hash[6] = 0x64f98fa7; + m_hash[7] = 0xbefa4fa4; +#endif +} + + +namespace +{ + inline uint32_t rotate(uint32_t a, uint32_t c) + { + return (a >> c) | (a << (32 - c)); + } + + inline uint32_t swap(uint32_t x) + { +#if defined(__GNUC__) || defined(__clang__) + return __builtin_bswap32(x); +#endif +#ifdef MSC_VER + return _byteswap_ulong(x); +#endif + + return (x >> 24) | + ((x >> 8) & 0x0000FF00) | + ((x << 8) & 0x00FF0000) | + (x << 24); + } + + // mix functions for processBlock() + inline uint32_t f1(uint32_t e, uint32_t f, uint32_t g) + { + uint32_t term1 = rotate(e, 6) ^ rotate(e, 11) ^ rotate(e, 25); + uint32_t term2 = (e & f) ^ (~e & g); //(g ^ (e & (f ^ g))) + return term1 + term2; + } + + inline uint32_t f2(uint32_t a, uint32_t b, uint32_t c) + { + uint32_t term1 = rotate(a, 2) ^ rotate(a, 13) ^ rotate(a, 22); + uint32_t term2 = ((a | b) & c) | (a & b); //(a & (b ^ c)) ^ (b & c); + return term1 + term2; + } +} + + +/// process 64 bytes +void SHA256::processBlock(const void* data) +{ + // get last hash + uint32_t a = m_hash[0]; + uint32_t b = m_hash[1]; + uint32_t c = m_hash[2]; + uint32_t d = m_hash[3]; + uint32_t e = m_hash[4]; + uint32_t f = m_hash[5]; + uint32_t g = m_hash[6]; + uint32_t h = m_hash[7]; + + // data represented as 16x 32-bit words + const uint32_t* input = (uint32_t*) data; + // convert to big endian + uint32_t words[64]; + int i; + for (i = 0; i < 16; i++) +#if defined(__BYTE_ORDER) && (__BYTE_ORDER != 0) && (__BYTE_ORDER == __BIG_ENDIAN) + words[i] = input[i]; +#else + words[i] = swap(input[i]); +#endif + + uint32_t x,y; // temporaries + + // first round + x = h + f1(e,f,g) + 0x428a2f98 + words[ 0]; y = f2(a,b,c); d += x; h = x + y; + x = g + f1(d,e,f) + 0x71374491 + words[ 1]; y = f2(h,a,b); c += x; g = x + y; + x = f + f1(c,d,e) + 0xb5c0fbcf + words[ 2]; y = f2(g,h,a); b += x; f = x + y; + x = e + f1(b,c,d) + 0xe9b5dba5 + words[ 3]; y = f2(f,g,h); a += x; e = x + y; + x = d + f1(a,b,c) + 0x3956c25b + words[ 4]; y = f2(e,f,g); h += x; d = x + y; + x = c + f1(h,a,b) + 0x59f111f1 + words[ 5]; y = f2(d,e,f); g += x; c = x + y; + x = b + f1(g,h,a) + 0x923f82a4 + words[ 6]; y = f2(c,d,e); f += x; b = x + y; + x = a + f1(f,g,h) + 0xab1c5ed5 + words[ 7]; y = f2(b,c,d); e += x; a = x + y; + + // secound round + x = h + f1(e,f,g) + 0xd807aa98 + words[ 8]; y = f2(a,b,c); d += x; h = x + y; + x = g + f1(d,e,f) + 0x12835b01 + words[ 9]; y = f2(h,a,b); c += x; g = x + y; + x = f + f1(c,d,e) + 0x243185be + words[10]; y = f2(g,h,a); b += x; f = x + y; + x = e + f1(b,c,d) + 0x550c7dc3 + words[11]; y = f2(f,g,h); a += x; e = x + y; + x = d + f1(a,b,c) + 0x72be5d74 + words[12]; y = f2(e,f,g); h += x; d = x + y; + x = c + f1(h,a,b) + 0x80deb1fe + words[13]; y = f2(d,e,f); g += x; c = x + y; + x = b + f1(g,h,a) + 0x9bdc06a7 + words[14]; y = f2(c,d,e); f += x; b = x + y; + x = a + f1(f,g,h) + 0xc19bf174 + words[15]; y = f2(b,c,d); e += x; a = x + y; + + // extend to 24 words + for (; i < 24; i++) + words[i] = words[i-16] + + (rotate(words[i-15], 7) ^ rotate(words[i-15], 18) ^ (words[i-15] >> 3)) + + words[i-7] + + (rotate(words[i- 2], 17) ^ rotate(words[i- 2], 19) ^ (words[i- 2] >> 10)); + + // third round + x = h + f1(e,f,g) + 0xe49b69c1 + words[16]; y = f2(a,b,c); d += x; h = x + y; + x = g + f1(d,e,f) + 0xefbe4786 + words[17]; y = f2(h,a,b); c += x; g = x + y; + x = f + f1(c,d,e) + 0x0fc19dc6 + words[18]; y = f2(g,h,a); b += x; f = x + y; + x = e + f1(b,c,d) + 0x240ca1cc + words[19]; y = f2(f,g,h); a += x; e = x + y; + x = d + f1(a,b,c) + 0x2de92c6f + words[20]; y = f2(e,f,g); h += x; d = x + y; + x = c + f1(h,a,b) + 0x4a7484aa + words[21]; y = f2(d,e,f); g += x; c = x + y; + x = b + f1(g,h,a) + 0x5cb0a9dc + words[22]; y = f2(c,d,e); f += x; b = x + y; + x = a + f1(f,g,h) + 0x76f988da + words[23]; y = f2(b,c,d); e += x; a = x + y; + + // extend to 32 words + for (; i < 32; i++) + words[i] = words[i-16] + + (rotate(words[i-15], 7) ^ rotate(words[i-15], 18) ^ (words[i-15] >> 3)) + + words[i-7] + + (rotate(words[i- 2], 17) ^ rotate(words[i- 2], 19) ^ (words[i- 2] >> 10)); + + // fourth round + x = h + f1(e,f,g) + 0x983e5152 + words[24]; y = f2(a,b,c); d += x; h = x + y; + x = g + f1(d,e,f) + 0xa831c66d + words[25]; y = f2(h,a,b); c += x; g = x + y; + x = f + f1(c,d,e) + 0xb00327c8 + words[26]; y = f2(g,h,a); b += x; f = x + y; + x = e + f1(b,c,d) + 0xbf597fc7 + words[27]; y = f2(f,g,h); a += x; e = x + y; + x = d + f1(a,b,c) + 0xc6e00bf3 + words[28]; y = f2(e,f,g); h += x; d = x + y; + x = c + f1(h,a,b) + 0xd5a79147 + words[29]; y = f2(d,e,f); g += x; c = x + y; + x = b + f1(g,h,a) + 0x06ca6351 + words[30]; y = f2(c,d,e); f += x; b = x + y; + x = a + f1(f,g,h) + 0x14292967 + words[31]; y = f2(b,c,d); e += x; a = x + y; + + // extend to 40 words + for (; i < 40; i++) + words[i] = words[i-16] + + (rotate(words[i-15], 7) ^ rotate(words[i-15], 18) ^ (words[i-15] >> 3)) + + words[i-7] + + (rotate(words[i- 2], 17) ^ rotate(words[i- 2], 19) ^ (words[i- 2] >> 10)); + + // fifth round + x = h + f1(e,f,g) + 0x27b70a85 + words[32]; y = f2(a,b,c); d += x; h = x + y; + x = g + f1(d,e,f) + 0x2e1b2138 + words[33]; y = f2(h,a,b); c += x; g = x + y; + x = f + f1(c,d,e) + 0x4d2c6dfc + words[34]; y = f2(g,h,a); b += x; f = x + y; + x = e + f1(b,c,d) + 0x53380d13 + words[35]; y = f2(f,g,h); a += x; e = x + y; + x = d + f1(a,b,c) + 0x650a7354 + words[36]; y = f2(e,f,g); h += x; d = x + y; + x = c + f1(h,a,b) + 0x766a0abb + words[37]; y = f2(d,e,f); g += x; c = x + y; + x = b + f1(g,h,a) + 0x81c2c92e + words[38]; y = f2(c,d,e); f += x; b = x + y; + x = a + f1(f,g,h) + 0x92722c85 + words[39]; y = f2(b,c,d); e += x; a = x + y; + + // extend to 48 words + for (; i < 48; i++) + words[i] = words[i-16] + + (rotate(words[i-15], 7) ^ rotate(words[i-15], 18) ^ (words[i-15] >> 3)) + + words[i-7] + + (rotate(words[i- 2], 17) ^ rotate(words[i- 2], 19) ^ (words[i- 2] >> 10)); + + // sixth round + x = h + f1(e,f,g) + 0xa2bfe8a1 + words[40]; y = f2(a,b,c); d += x; h = x + y; + x = g + f1(d,e,f) + 0xa81a664b + words[41]; y = f2(h,a,b); c += x; g = x + y; + x = f + f1(c,d,e) + 0xc24b8b70 + words[42]; y = f2(g,h,a); b += x; f = x + y; + x = e + f1(b,c,d) + 0xc76c51a3 + words[43]; y = f2(f,g,h); a += x; e = x + y; + x = d + f1(a,b,c) + 0xd192e819 + words[44]; y = f2(e,f,g); h += x; d = x + y; + x = c + f1(h,a,b) + 0xd6990624 + words[45]; y = f2(d,e,f); g += x; c = x + y; + x = b + f1(g,h,a) + 0xf40e3585 + words[46]; y = f2(c,d,e); f += x; b = x + y; + x = a + f1(f,g,h) + 0x106aa070 + words[47]; y = f2(b,c,d); e += x; a = x + y; + + // extend to 56 words + for (; i < 56; i++) + words[i] = words[i-16] + + (rotate(words[i-15], 7) ^ rotate(words[i-15], 18) ^ (words[i-15] >> 3)) + + words[i-7] + + (rotate(words[i- 2], 17) ^ rotate(words[i- 2], 19) ^ (words[i- 2] >> 10)); + + // seventh round + x = h + f1(e,f,g) + 0x19a4c116 + words[48]; y = f2(a,b,c); d += x; h = x + y; + x = g + f1(d,e,f) + 0x1e376c08 + words[49]; y = f2(h,a,b); c += x; g = x + y; + x = f + f1(c,d,e) + 0x2748774c + words[50]; y = f2(g,h,a); b += x; f = x + y; + x = e + f1(b,c,d) + 0x34b0bcb5 + words[51]; y = f2(f,g,h); a += x; e = x + y; + x = d + f1(a,b,c) + 0x391c0cb3 + words[52]; y = f2(e,f,g); h += x; d = x + y; + x = c + f1(h,a,b) + 0x4ed8aa4a + words[53]; y = f2(d,e,f); g += x; c = x + y; + x = b + f1(g,h,a) + 0x5b9cca4f + words[54]; y = f2(c,d,e); f += x; b = x + y; + x = a + f1(f,g,h) + 0x682e6ff3 + words[55]; y = f2(b,c,d); e += x; a = x + y; + + // extend to 64 words + for (; i < 64; i++) + words[i] = words[i-16] + + (rotate(words[i-15], 7) ^ rotate(words[i-15], 18) ^ (words[i-15] >> 3)) + + words[i-7] + + (rotate(words[i- 2], 17) ^ rotate(words[i- 2], 19) ^ (words[i- 2] >> 10)); + + // eigth round + x = h + f1(e,f,g) + 0x748f82ee + words[56]; y = f2(a,b,c); d += x; h = x + y; + x = g + f1(d,e,f) + 0x78a5636f + words[57]; y = f2(h,a,b); c += x; g = x + y; + x = f + f1(c,d,e) + 0x84c87814 + words[58]; y = f2(g,h,a); b += x; f = x + y; + x = e + f1(b,c,d) + 0x8cc70208 + words[59]; y = f2(f,g,h); a += x; e = x + y; + x = d + f1(a,b,c) + 0x90befffa + words[60]; y = f2(e,f,g); h += x; d = x + y; + x = c + f1(h,a,b) + 0xa4506ceb + words[61]; y = f2(d,e,f); g += x; c = x + y; + x = b + f1(g,h,a) + 0xbef9a3f7 + words[62]; y = f2(c,d,e); f += x; b = x + y; + x = a + f1(f,g,h) + 0xc67178f2 + words[63]; y = f2(b,c,d); e += x; a = x + y; + + // update hash + m_hash[0] += a; + m_hash[1] += b; + m_hash[2] += c; + m_hash[3] += d; + m_hash[4] += e; + m_hash[5] += f; + m_hash[6] += g; + m_hash[7] += h; +} + + +/// add arbitrary number of bytes +void SHA256::add(const void* data, size_t numBytes) +{ + const uint8_t* current = (const uint8_t*) data; + + if (m_bufferSize > 0) + { + while (numBytes > 0 && m_bufferSize < BlockSize) + { + m_buffer[m_bufferSize++] = *current++; + numBytes--; + } + } + + // full buffer + if (m_bufferSize == BlockSize) + { + processBlock(m_buffer); + m_numBytes += BlockSize; + m_bufferSize = 0; + } + + // no more data ? + if (numBytes == 0) + return; + + // process full blocks + while (numBytes >= BlockSize) + { + processBlock(current); + current += BlockSize; + m_numBytes += BlockSize; + numBytes -= BlockSize; + } + + // keep remaining bytes in buffer + while (numBytes > 0) + { + m_buffer[m_bufferSize++] = *current++; + numBytes--; + } +} + + +/// process final block, less than 64 bytes +void SHA256::processBuffer() +{ + // the input bytes are considered as bits strings, where the first bit is the most significant bit of the byte + + // - append "1" bit to message + // - append "0" bits until message length in bit mod 512 is 448 + // - append length as 64 bit integer + + // number of bits + size_t paddedLength = m_bufferSize * 8; + + // plus one bit set to 1 (always appended) + paddedLength++; + + // number of bits must be (numBits % 512) = 448 + size_t lower11Bits = paddedLength & 511; + if (lower11Bits <= 448) + paddedLength += 448 - lower11Bits; + else + paddedLength += 512 + 448 - lower11Bits; + // convert from bits to bytes + paddedLength /= 8; + + // only needed if additional data flows over into a second block + unsigned char extra[BlockSize]; + + // append a "1" bit, 128 => binary 10000000 + if (m_bufferSize < BlockSize) + m_buffer[m_bufferSize] = 128; + else + extra[0] = 128; + + size_t i; + for (i = m_bufferSize + 1; i < BlockSize; i++) + m_buffer[i] = 0; + for (; i < paddedLength; i++) + extra[i - BlockSize] = 0; + + // add message length in bits as 64 bit number + uint64_t msgBits = 8 * (m_numBytes + m_bufferSize); + // find right position + unsigned char* addLength; + if (paddedLength < BlockSize) + addLength = m_buffer + paddedLength; + else + addLength = extra + paddedLength - BlockSize; + + // must be big endian + *addLength++ = (unsigned char)((msgBits >> 56) & 0xFF); + *addLength++ = (unsigned char)((msgBits >> 48) & 0xFF); + *addLength++ = (unsigned char)((msgBits >> 40) & 0xFF); + *addLength++ = (unsigned char)((msgBits >> 32) & 0xFF); + *addLength++ = (unsigned char)((msgBits >> 24) & 0xFF); + *addLength++ = (unsigned char)((msgBits >> 16) & 0xFF); + *addLength++ = (unsigned char)((msgBits >> 8) & 0xFF); + *addLength = (unsigned char)( msgBits & 0xFF); + + // process blocks + processBlock(m_buffer); + // flowed over into a second block ? + if (paddedLength > BlockSize) + processBlock(extra); +} + + +/// return latest hash as 64 hex characters +std::string SHA256::getHash() +{ + // compute hash (as raw bytes) + unsigned char rawHash[HashBytes]; + getHash(rawHash); + + // convert to hex string + std::string result; + result.reserve(2 * HashBytes); + for (int i = 0; i < HashBytes; i++) + { + static const char dec2hex[16+1] = "0123456789abcdef"; + result += dec2hex[(rawHash[i] >> 4) & 15]; + result += dec2hex[ rawHash[i] & 15]; + } + + return result; +} + + +/// return latest hash as bytes +void SHA256::getHash(unsigned char buffer[SHA256::HashBytes]) +{ + // save old hash if buffer is partially filled + uint32_t oldHash[HashValues]; + for (int i = 0; i < HashValues; i++) + oldHash[i] = m_hash[i]; + + // process remaining bytes + processBuffer(); + + unsigned char* current = buffer; + for (int i = 0; i < HashValues; i++) + { + *current++ = (m_hash[i] >> 24) & 0xFF; + *current++ = (m_hash[i] >> 16) & 0xFF; + *current++ = (m_hash[i] >> 8) & 0xFF; + *current++ = m_hash[i] & 0xFF; + + // restore old hash + m_hash[i] = oldHash[i]; + } +} + + +/// compute SHA256 of a memory block +std::string SHA256::operator()(const void* data, size_t numBytes) +{ + reset(); + add(data, numBytes); + return getHash(); +} + + +/// compute SHA256 of a string, excluding final zero +std::string SHA256::operator()(const std::string& text) +{ + reset(); + add(text.c_str(), text.size()); + return getHash(); +} diff -pruN 1.4-2/3rd-party/hash-library/sha256.h 2.5-1/3rd-party/hash-library/sha256.h --- 1.4-2/3rd-party/hash-library/sha256.h 1970-01-01 00:00:00.000000000 +0000 +++ 2.5-1/3rd-party/hash-library/sha256.h 2022-06-15 07:33:00.000000000 +0000 @@ -0,0 +1,78 @@ +// ////////////////////////////////////////////////////////// +// sha256.h +// Copyright (c) 2014,2015 Stephan Brumme. All rights reserved. +// see http://create.stephan-brumme.com/disclaimer.html +// + +#pragma once + +//#include "hash.h" +#include + +// define fixed size integer types +#ifdef _MSC_VER +// Windows +typedef unsigned __int8 uint8_t; +typedef unsigned __int32 uint32_t; +typedef unsigned __int64 uint64_t; +#else +// GCC +#include +#endif + + +/// compute SHA256 hash +/** Usage: + SHA256 sha256; + std::string myHash = sha256("Hello World"); // std::string + std::string myHash2 = sha256("How are you", 11); // arbitrary data, 11 bytes + + // or in a streaming fashion: + + SHA256 sha256; + while (more data available) + sha256.add(pointer to fresh data, number of new bytes); + std::string myHash3 = sha256.getHash(); + */ +class SHA256 //: public Hash +{ +public: + /// split into 64 byte blocks (=> 512 bits), hash is 32 bytes long + enum { BlockSize = 512 / 8, HashBytes = 32 }; + + /// same as reset() + SHA256(); + + /// compute SHA256 of a memory block + std::string operator()(const void* data, size_t numBytes); + /// compute SHA256 of a string, excluding final zero + std::string operator()(const std::string& text); + + /// add arbitrary number of bytes + void add(const void* data, size_t numBytes); + + /// return latest hash as 64 hex characters + std::string getHash(); + /// return latest hash as bytes + void getHash(unsigned char buffer[HashBytes]); + + /// restart + void reset(); + +private: + /// process 64 bytes + void processBlock(const void* data); + /// process everything left in the internal buffer + void processBuffer(); + + /// size of processed data in bytes + uint64_t m_numBytes; + /// valid bytes in m_buffer + size_t m_bufferSize; + /// bytes not processed yet + uint8_t m_buffer[BlockSize]; + + enum { HashValues = HashBytes / 4 }; + /// hash, stored as integers + uint32_t m_hash[HashValues]; +}; diff -pruN 1.4-2/3rd-party/hash-library/sha3.cpp 2.5-1/3rd-party/hash-library/sha3.cpp --- 1.4-2/3rd-party/hash-library/sha3.cpp 1970-01-01 00:00:00.000000000 +0000 +++ 2.5-1/3rd-party/hash-library/sha3.cpp 2022-06-15 07:33:00.000000000 +0000 @@ -0,0 +1,300 @@ +// ////////////////////////////////////////////////////////// +// sha3.cpp +// Copyright (c) 2014,2015 Stephan Brumme. All rights reserved. +// see http://create.stephan-brumme.com/disclaimer.html +// + +#include "sha3.h" + +// big endian architectures need #define __BYTE_ORDER __BIG_ENDIAN +#ifndef _MSC_VER +#include +#endif + +#include + + +/// same as reset() +SHA3::SHA3(Bits bits) +: m_blockSize(200 - 2 * (bits / 8)), + m_bits(bits) +{ + reset(); +} + + +/// restart +void SHA3::reset() +{ + for (size_t i = 0; i < StateSize; i++) + m_hash[i] = 0; + + m_numBytes = 0; + m_bufferSize = 0; +} + + +/// constants and local helper functions +namespace +{ + const unsigned int Rounds = 24; + const uint64_t XorMasks[Rounds] = + { + 0x0000000000000001ULL, 0x0000000000008082ULL, 0x800000000000808aULL, + 0x8000000080008000ULL, 0x000000000000808bULL, 0x0000000080000001ULL, + 0x8000000080008081ULL, 0x8000000000008009ULL, 0x000000000000008aULL, + 0x0000000000000088ULL, 0x0000000080008009ULL, 0x000000008000000aULL, + 0x000000008000808bULL, 0x800000000000008bULL, 0x8000000000008089ULL, + 0x8000000000008003ULL, 0x8000000000008002ULL, 0x8000000000000080ULL, + 0x000000000000800aULL, 0x800000008000000aULL, 0x8000000080008081ULL, + 0x8000000000008080ULL, 0x0000000080000001ULL, 0x8000000080008008ULL + }; + + /// rotate left and wrap around to the right + inline uint64_t rotateLeft(uint64_t x, uint8_t numBits) + { + return (x << numBits) | (x >> (64 - numBits)); + } + + /// convert litte vs big endian + inline uint64_t swap(uint64_t x) + { +#if defined(__GNUC__) || defined(__clang__) + return __builtin_bswap64(x); +#endif +#ifdef _MSC_VER + return _byteswap_uint64(x); +#endif + + return (x >> 56) | + ((x >> 40) & 0x000000000000FF00ULL) | + ((x >> 24) & 0x0000000000FF0000ULL) | + ((x >> 8) & 0x00000000FF000000ULL) | + ((x << 8) & 0x000000FF00000000ULL) | + ((x << 24) & 0x0000FF0000000000ULL) | + ((x << 40) & 0x00FF000000000000ULL) | + (x << 56); + } + + + /// return x % 5 for 0 <= x <= 9 + unsigned int mod5(unsigned int x) + { + if (x < 5) + return x; + + return x - 5; + } +} + + +/// process a full block +void SHA3::processBlock(const void* data) +{ +#if defined(__BYTE_ORDER) && (__BYTE_ORDER != 0) && (__BYTE_ORDER == __BIG_ENDIAN) +#define LITTLEENDIAN(x) swap(x) +#else +#define LITTLEENDIAN(x) (x) +#endif + + const uint64_t* data64 = (const uint64_t*) data; + // mix data into state + for (unsigned int i = 0; i < m_blockSize / 8; i++) + m_hash[i] ^= LITTLEENDIAN(data64[i]); + + // re-compute state + for (unsigned int round = 0; round < Rounds; round++) + { + // Theta + uint64_t coefficients[5]; + for (unsigned int i = 0; i < 5; i++) + coefficients[i] = m_hash[i] ^ m_hash[i + 5] ^ m_hash[i + 10] ^ m_hash[i + 15] ^ m_hash[i + 20]; + + for (unsigned int i = 0; i < 5; i++) + { + uint64_t one = coefficients[mod5(i + 4)] ^ rotateLeft(coefficients[mod5(i + 1)], 1); + m_hash[i ] ^= one; + m_hash[i + 5] ^= one; + m_hash[i + 10] ^= one; + m_hash[i + 15] ^= one; + m_hash[i + 20] ^= one; + } + + // temporary + uint64_t one; + + // Rho Pi + uint64_t last = m_hash[1]; + one = m_hash[10]; m_hash[10] = rotateLeft(last, 1); last = one; + one = m_hash[ 7]; m_hash[ 7] = rotateLeft(last, 3); last = one; + one = m_hash[11]; m_hash[11] = rotateLeft(last, 6); last = one; + one = m_hash[17]; m_hash[17] = rotateLeft(last, 10); last = one; + one = m_hash[18]; m_hash[18] = rotateLeft(last, 15); last = one; + one = m_hash[ 3]; m_hash[ 3] = rotateLeft(last, 21); last = one; + one = m_hash[ 5]; m_hash[ 5] = rotateLeft(last, 28); last = one; + one = m_hash[16]; m_hash[16] = rotateLeft(last, 36); last = one; + one = m_hash[ 8]; m_hash[ 8] = rotateLeft(last, 45); last = one; + one = m_hash[21]; m_hash[21] = rotateLeft(last, 55); last = one; + one = m_hash[24]; m_hash[24] = rotateLeft(last, 2); last = one; + one = m_hash[ 4]; m_hash[ 4] = rotateLeft(last, 14); last = one; + one = m_hash[15]; m_hash[15] = rotateLeft(last, 27); last = one; + one = m_hash[23]; m_hash[23] = rotateLeft(last, 41); last = one; + one = m_hash[19]; m_hash[19] = rotateLeft(last, 56); last = one; + one = m_hash[13]; m_hash[13] = rotateLeft(last, 8); last = one; + one = m_hash[12]; m_hash[12] = rotateLeft(last, 25); last = one; + one = m_hash[ 2]; m_hash[ 2] = rotateLeft(last, 43); last = one; + one = m_hash[20]; m_hash[20] = rotateLeft(last, 62); last = one; + one = m_hash[14]; m_hash[14] = rotateLeft(last, 18); last = one; + one = m_hash[22]; m_hash[22] = rotateLeft(last, 39); last = one; + one = m_hash[ 9]; m_hash[ 9] = rotateLeft(last, 61); last = one; + one = m_hash[ 6]; m_hash[ 6] = rotateLeft(last, 20); last = one; + m_hash[ 1] = rotateLeft(last, 44); + + // Chi + for (unsigned int j = 0; j < StateSize; j += 5) + { + // temporaries + uint64_t one = m_hash[j]; + uint64_t two = m_hash[j + 1]; + + m_hash[j] ^= m_hash[j + 2] & ~two; + m_hash[j + 1] ^= m_hash[j + 3] & ~m_hash[j + 2]; + m_hash[j + 2] ^= m_hash[j + 4] & ~m_hash[j + 3]; + m_hash[j + 3] ^= one & ~m_hash[j + 4]; + m_hash[j + 4] ^= two & ~one; + } + + // Iota + m_hash[0] ^= XorMasks[round]; + } +} + + +/// add arbitrary number of bytes +void SHA3::add(const void* data, size_t numBytes) +{ + const uint8_t* current = (const uint8_t*) data; + + // copy data to buffer + if (m_bufferSize > 0) + { + while (numBytes > 0 && m_bufferSize < m_blockSize) + { + m_buffer[m_bufferSize++] = *current++; + numBytes--; + } + } + + // full buffer + if (m_bufferSize == m_blockSize) + { + processBlock((void*)m_buffer); + m_numBytes += m_blockSize; + m_bufferSize = 0; + } + + // no more data ? + if (numBytes == 0) + return; + + // process full blocks + while (numBytes >= m_blockSize) + { + processBlock(current); + current += m_blockSize; + m_numBytes += m_blockSize; + numBytes -= m_blockSize; + } + + // keep remaining bytes in buffer + while (numBytes > 0) + { + m_buffer[m_bufferSize++] = *current++; + numBytes--; + } +} + + +/// process everything left in the internal buffer +void SHA3::processBuffer() +{ + // add padding + size_t offset = m_bufferSize; + // add a "1" byte + m_buffer[offset++] = 0x06; + // fill with zeros + while (offset < m_blockSize) + m_buffer[offset++] = 0; + + // and add a single set bit + m_buffer[offset - 1] |= 0x80; + + processBlock(m_buffer); +} + + +/// return latest hash as 16 hex characters +std::string SHA3::getHash() +{ + // save hash state + uint64_t oldHash[StateSize]; + for (unsigned int i = 0; i < StateSize; i++) + oldHash[i] = m_hash[i]; + + // process remaining bytes + processBuffer(); + + // convert hash to string + static const char dec2hex[16 + 1] = "0123456789abcdef"; + + // number of significant elements in hash (uint64_t) + unsigned int hashLength = m_bits / 64; + + std::string result; + result.reserve(m_bits / 4); + for (unsigned int i = 0; i < hashLength; i++) + for (unsigned int j = 0; j < 8; j++) // 64 bits => 8 bytes + { + // convert a byte to hex + unsigned char oneByte = (unsigned char) (m_hash[i] >> (8 * j)); + result += dec2hex[oneByte >> 4]; + result += dec2hex[oneByte & 15]; + } + + // SHA3-224's last entry in m_hash provides only 32 bits instead of 64 bits + unsigned int remainder = m_bits - hashLength * 64; + unsigned int processed = 0; + while (processed < remainder) + { + // convert a byte to hex + unsigned char oneByte = (unsigned char) (m_hash[hashLength] >> processed); + result += dec2hex[oneByte >> 4]; + result += dec2hex[oneByte & 15]; + + processed += 8; + } + + // restore state + for (unsigned int i = 0; i < StateSize; i++) + m_hash[i] = oldHash[i]; + + return result; +} + + +/// compute SHA3 of a memory block +std::string SHA3::operator()(const void* data, size_t numBytes) +{ + reset(); + add(data, numBytes); + return getHash(); +} + + +/// compute SHA3 of a string, excluding final zero +std::string SHA3::operator()(const std::string& text) +{ + reset(); + add(text.c_str(), text.size()); + return getHash(); +} diff -pruN 1.4-2/3rd-party/hash-library/sha3.h 2.5-1/3rd-party/hash-library/sha3.h --- 1.4-2/3rd-party/hash-library/sha3.h 1970-01-01 00:00:00.000000000 +0000 +++ 2.5-1/3rd-party/hash-library/sha3.h 2022-06-15 07:33:00.000000000 +0000 @@ -0,0 +1,81 @@ +// ////////////////////////////////////////////////////////// +// sha3.h +// Copyright (c) 2014,2015 Stephan Brumme. All rights reserved. +// see http://create.stephan-brumme.com/disclaimer.html +// + +#pragma once + +//#include "hash.h" +#include + +// define fixed size integer types +#ifdef _MSC_VER +// Windows +typedef unsigned __int8 uint8_t; +typedef unsigned __int64 uint64_t; +#else +// GCC +#include +#endif + + +/// compute SHA3 hash +/** Usage: + SHA3 sha3; + std::string myHash = sha3("Hello World"); // std::string + std::string myHash2 = sha3("How are you", 11); // arbitrary data, 11 bytes + + // or in a streaming fashion: + + SHA3 sha3; + while (more data available) + sha3.add(pointer to fresh data, number of new bytes); + std::string myHash3 = sha3.getHash(); + */ +class SHA3 //: public Hash +{ +public: + /// algorithm variants + enum Bits { Bits224 = 224, Bits256 = 256, Bits384 = 384, Bits512 = 512 }; + + /// same as reset() + explicit SHA3(Bits bits = Bits256); + + /// compute hash of a memory block + std::string operator()(const void* data, size_t numBytes); + /// compute hash of a string, excluding final zero + std::string operator()(const std::string& text); + + /// add arbitrary number of bytes + void add(const void* data, size_t numBytes); + + /// return latest hash as hex characters + std::string getHash(); + + /// restart + void reset(); + +private: + /// process a full block + void processBlock(const void* data); + /// process everything left in the internal buffer + void processBuffer(); + + /// 1600 bits, stored as 25x64 bit, BlockSize is no more than 1152 bits (Keccak224) + enum { StateSize = 1600 / (8 * 8), + MaxBlockSize = 200 - 2 * (224 / 8) }; + + /// hash + uint64_t m_hash[StateSize]; + /// size of processed data in bytes + uint64_t m_numBytes; + /// block size (less or equal to MaxBlockSize) + size_t m_blockSize; + /// valid bytes in m_buffer + size_t m_bufferSize; + /// bytes not processed yet + uint8_t m_buffer[MaxBlockSize]; + /// variant + Bits m_bits; +}; diff -pruN 1.4-2/build.bat 2.5-1/build.bat --- 1.4-2/build.bat 1970-01-01 00:00:00.000000000 +0000 +++ 2.5-1/build.bat 2022-06-15 07:33:00.000000000 +0000 @@ -0,0 +1,116 @@ +@ECHO OFF + +SET PREFIX=C:\Xilinx\XRT\ext + +IF "%1" == "-clean" ( + GOTO Clean +) + +IF "%1" == "-help" ( + GOTO Help +) + +IF "%1" == "-debug" ( + GOTO DebugBuild +) + +IF "%1" == "-release" ( + GOTO ReleaseBuild +) + +IF "%1" == "-all" ( + CALL:DebugBuild + IF errorlevel 1 (exit /B %errorlevel%) + + CALL:ReleaseBuild + IF errorlevel 1 (exit /B %errorlevel%) + + goto:EOF +) + +IF "%1" == "" ( + + CALL:ReleaseBuild + IF errorlevel 1 (exit /B %errorlevel%) + + GOTO:EOF +) + +ECHO Unknown option: %1 +GOTO Help + + +REM -------------------------------------------------------------------------- +:Help +ECHO. +ECHO Usage: build.bat [options] +ECHO. +ECHO [-help] - List this help +ECHO [-clean] - Remove build directories +ECHO [-debug] - Creates a debug build +ECHO [-release] - Creates a release build +ECHO [-all] - Creates a release build and a debug build +ECHO. +GOTO:EOF + +REM -------------------------------------------------------------------------- +:Clean +IF EXIST WDebug ( + ECHO Removing 'WDebug' directory... + rmdir /S /Q WDebug +) +IF EXIST WRelease ( + ECHO Removing 'WRelease' directory... + rmdir /S /Q WRelease +) +GOTO:EOF + +REM -------------------------------------------------------------------------- +:DebugBuild +ECHO ====================== Windows Debug Build ============================ +MKDIR WDebug +PUSHD WDebug + +cmake -G "Visual Studio 15 2017 Win64" -DCMAKE_PREFIX_PATH=%PREFIX% -DCMAKE_INSTALL_PREFIX=%PREFIX% -DCMAKE_BUILD_TYPE=Debug -DCMAKE_EXPORT_COMPILE_COMMANDS=ON .. +IF errorlevel 1 (POPD & exit /B %errorlevel%) + +cmake --build . --verbose --config Debug +IF errorlevel 1 (POPD & exit /B %errorlevel%) + +cmake --build . --verbose --config Debug --target install +IF errorlevel 1 (POPD & exit /B %errorlevel%) + +ECHO ====================== Zipping up Installation Build ============================ +cpack -G ZIP -C Debug + +ECHO ====================== Creating MSI Archive ============================ +cpack -G WIX -C Debug + +POPD +GOTO:EOF + +REM -------------------------------------------------------------------------- +:ReleaseBuild +ECHO ====================== Windows Release Build ============================ +MKDIR WRelease +PUSHD WRelease + +cmake -G "Visual Studio 15 2017 Win64" -DCMAKE_PREFIX_PATH=%PREFIX% -DCMAKE_INSTALL_PREFIX=%PREFIX% -DCMAKE_BUILD_TYPE=Release -DCMAKE_EXPORT_COMPILE_COMMANDS=ON .. +IF errorlevel 1 (POPD & exit /B %errorlevel%) + +cmake --build . --verbose --config Release +IF errorlevel 1 (POPD & exit /B %errorlevel%) + +cmake --build . --verbose --config Release --target install +IF errorlevel 1 (POPD & exit /B %errorlevel%) + +ECHO ====================== Zipping up Installation Build ============================ +cpack -G ZIP -C Release + +ECHO ====================== Creating MSI Archive ============================ +cpack -G WIX -C Release + +POPD +GOTO:EOF + + diff -pruN 1.4-2/cmake/VitisCommon.cmake 2.5-1/cmake/VitisCommon.cmake --- 1.4-2/cmake/VitisCommon.cmake 2021-07-27 05:41:52.000000000 +0000 +++ 2.5-1/cmake/VitisCommon.cmake 2022-06-15 07:33:00.000000000 +0000 @@ -16,14 +16,17 @@ set (CMAKE_CXX_STANDARD 14) set (CMAKE_C_STANDARD 99) -set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wall -Werror -ggdb -O0 -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=0 -fno-inline") -set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3 -Wall -Werror") -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=gnu++14 -Wall -Werror") -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Werror") -set(CMAKE_EXE "${CMAKE_C_FLAGS} -Wall -Werror") -set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--no-undefined") - -set(CMAKE_MACOSX_RPATH 1) +if (MSVC) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Zc:__cplusplus") +else(MSVC) + set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wall -Werror -ggdb -O0 -fno-inline") + set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3 -Wall -Werror") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Werror") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Werror") + set(CMAKE_EXE "${CMAKE_C_FLAGS} -Wall -Werror") + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--no-undefined") + set(CMAKE_MACOSX_RPATH 1) +endif(MSVC) if(CMAKE_BUILD_TYPE MATCHES "Release") ADD_DEFINITIONS(-DUNI_LOG_NDEBUG) diff -pruN 1.4-2/cmake/vitis_version.c.in 2.5-1/cmake/vitis_version.c.in --- 1.4-2/cmake/vitis_version.c.in 2021-07-27 05:41:52.000000000 +0000 +++ 2.5-1/cmake/vitis_version.c.in 2022-06-15 07:33:00.000000000 +0000 @@ -21,4 +21,3 @@ static char @PROJECT_NAME@_version[] = " const char* xilinx_version() { return @PROJECT_NAME@_version; } - diff -pruN 1.4-2/CMakeLists.txt 2.5-1/CMakeLists.txt --- 1.4-2/CMakeLists.txt 2021-07-27 05:41:52.000000000 +0000 +++ 2.5-1/CMakeLists.txt 2022-06-15 07:33:00.000000000 +0000 @@ -1,23 +1,26 @@ # # Copyright 2019 Xilinx Inc. # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. # cmake_minimum_required(VERSION 3.9) -project(xir VERSION 1.4.0 LANGUAGES C CXX) +project( + xir + VERSION 2.5.0 + LANGUAGES C CXX) -include(${CMAKE_SOURCE_DIR}/cmake/VitisCommon.cmake) +include(${PROJECT_SOURCE_DIR}/cmake/VitisCommon.cmake) option(BUILD_CONTRIB "build contrib lib" OFF) option(BUILD_TEST "build test bin" OFF) @@ -26,31 +29,33 @@ option(BUILD_PYTHON "build python interf option(INSTALL_HOME "install python lib in cmake install path" OFF) option(INSTALL_USER "install python lib in user space" OFF) -find_package(unilog REQUIRED) -find_package(OpenSSL REQUIRED) +# find_package(glog REQUIRED) +if(NOT TARGET unilog::unilog) + find_package(unilog REQUIRED) +endif() + find_package(Boost REQUIRED) find_package(Protobuf REQUIRED) +find_package(Threads REQUIRED) add_subdirectory(src) add_subdirectory(tools) file(RELATIVE_PATH REL_INCLUDE_DIR - ${CMAKE_INSTALL_PREFIX}/share/cmake/${PROJECT_NAME} - ${CMAKE_INSTALL_PREFIX}/include) + ${CMAKE_INSTALL_PREFIX}/share/cmake/${PROJECT_NAME} + ${CMAKE_INSTALL_PREFIX}/include) set(CONF_INCLUDE_DIRS "\${${PROJECT_NAME}_CMAKE_DIR}/${REL_INCLUDE_DIR}") configure_file(cmake/config.cmake.in - "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}-config.cmake" - @ONLY) + "${CMAKE_BINARY_DIR}/${PROJECT_NAME}-config.cmake" @ONLY) write_basic_package_version_file( "${CMAKE_BINARY_DIR}/${PROJECT_NAME}-config-version.cmake" VERSION ${PROJECT_VERSION} COMPATIBILITY AnyNewerVersion) -install(FILES - "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}-config.cmake" - "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}-config-version.cmake" - DESTINATION share/cmake/${PROJECT_NAME}) +install(FILES "${CMAKE_BINARY_DIR}/${PROJECT_NAME}-config.cmake" + "${CMAKE_BINARY_DIR}/${PROJECT_NAME}-config-version.cmake" + DESTINATION share/cmake/${PROJECT_NAME}) if(BUILD_TEST) add_subdirectory(test) @@ -61,10 +66,11 @@ if(BUILD_DOC) endif(BUILD_DOC) set(CPACK_PACKAGE_NAME "libxir") -set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Xilinx Intermediate Representation for deep learning algorithms") -set(CPACK_PACKAGE_DESCRIPTION_FILE "${CMAKE_SOURCE_DIR}/README.md") -set(CPACK_RESOURCE_FILE_README "${CMAKE_SOURCE_DIR}/README.md") -set(CPACK_RESOURCE_FILE_LICENSE "${CMAKE_SOURCE_DIR}/Copyright.txt") +set(CPACK_PACKAGE_DESCRIPTION_SUMMARY + "Xilinx Intermediate Representation for deep learning algorithms") +set(CPACK_PACKAGE_DESCRIPTION_FILE "${PROJECT_SOURCE_DIR}/README.md") +set(CPACK_RESOURCE_FILE_README "${PROJECT_SOURCE_DIR}/README.md") +set(CPACK_RESOURCE_FILE_LICENSE "${PROJECT_SOURCE_DIR}/Copyright.txt") set(CPACK_PACKAGE_VENDOR "Humanity") set(CPACK_PACKAGE_VERSION_MAJOR ${CMAKE_PROJECT_VERSION_MAJOR}) set(CPACK_PACKAGE_VERSION_MINOR ${CMAKE_PROJECT_VERSION_MINOR}) @@ -101,7 +107,7 @@ if(DEFINED ENV{BUILD_NUMBER}) endif() set(CPACK_RPM_PACKAGE_ARCHITECTURE ${CMAKE_SYSTEM_PROCESSOR}) if(CMAKE_SYSTEM_PROCESSOR STREQUAL "cortexa72-cortexa53") - set(CPACK_RPM_PACKAGE_ARCHITECTURE "aarch64") + set(CPACK_RPM_PACKAGE_ARCHITECTURE "aarch64") endif() include(CPack) diff -pruN 1.4-2/debian/changelog 2.5-1/debian/changelog --- 1.4-2/debian/changelog 2021-10-21 02:43:54.000000000 +0000 +++ 2.5-1/debian/changelog 2022-08-03 07:38:14.000000000 +0000 @@ -1,3 +1,71 @@ +xir (2.5-1) unstable; urgency=medium + + * New upstream release. + + -- Nobuhiro Iwamatsu Wed, 03 Aug 2022 16:38:14 +0900 + +xir (2.0-1) experimental; urgency=medium + + * New upstream release. + * Update soname to 2. + * Update d/patches. + - Remove Fix-FTBFS-with-Werror-range-loop-construct-Closes-99.patch. + - Refresh patches. + * Update d/control. + - Bump Standards-Version to 4.6.0.1. + * Update d/rules + - Drop DEB_LDFLAGS_MAINT_APPEND. + Fix lintian:debian-rules-uses-as-needed-linker-flag. + + -- Nobuhiro Iwamatsu Tue, 07 Jun 2022 07:21:57 +0900 + +xir (1.4.1-5.1) unstable; urgency=medium + + * Non-maintainer upload. + * Don't build with -Werror. (Closes: #1006582) + + -- Adrian Bunk Sun, 22 May 2022 22:34:37 +0300 + +xir (1.4.1-5) unstable; urgency=medium + + [ Vagrant Cascadian ] + * Pass -DCMAKE_BUILD_RPATH_USE_ORIGIN=ON via dh_auto_configure override. + (Closes: #1005420) + + [ Punit Agrawal ] + * Add a patch to remove build date from the embedded version + (Closes: #1005421) + + -- Punit Agrawal Sun, 13 Feb 2022 16:36:14 +0900 + +xir (1.4.1-4) unstable; urgency=medium + + * Update d/control. + - Change Maintainer and Uploaders. Set team email address to Maintainer. + + -- Nobuhiro Iwamatsu Fri, 21 Jan 2022 04:35:15 +0900 + +xir (1.4.1-3) unstable; urgency=medium + + * Upload to unstable. + + -- Nobuhiro Iwamatsu Thu, 20 Jan 2022 10:04:28 +0900 + +xir (1.4.1-2) experimental; urgency=medium + + * Update d/control: + - Add armel to Support architecture. + + -- Nobuhiro Iwamatsu Wed, 19 Jan 2022 11:42:49 +0900 + +xir (1.4.1-1) unstable; urgency=medium + + * New upstream released. + * Update d/control + - Update Maintainer and Uploaders. + + -- Nobuhiro Iwamatsu Wed, 22 Dec 2021 20:45:12 +0900 + xir (1.4-2) unstable; urgency=medium * Update d/control. diff -pruN 1.4-2/debian/control 2.5-1/debian/control --- 1.4-2/debian/control 2021-10-21 02:43:54.000000000 +0000 +++ 2.5-1/debian/control 2022-08-03 07:38:14.000000000 +0000 @@ -1,22 +1,23 @@ Source: xir -Section: libs +Maintainer: Debian Xilinx Package Maintainers +Uploaders: Punit Agrawal , + Nobuhiro Iwamatsu Priority: optional -Maintainer: Punit Agrawal -Uploaders: Nobuhiro Iwamatsu +Section: libs Build-Depends: cmake, debhelper-compat (= 13), libboost-dev, libprotobuf-dev, libssl-dev, - libunilog-dev, + libunilog-dev (>= 2.5), protobuf-compiler -Standards-Version: 4.5.0 +Standards-Version: 4.6.1 Rules-Requires-Root: no Homepage: https://github.com/Xilinx/Vitis-AI Vcs-Git: https://salsa.debian.org/xilinx-packages-team/vitis-ai/xir.git Vcs-Browser: https://salsa.debian.org/xilinx-packages-team/vitis-ai/xir -Package: libxir1 -Architecture: amd64 arm64 armhf +Package: libxir2 +Architecture: amd64 arm64 armhf armel Multi-Arch: same Depends: ${misc:Depends}, ${shlibs:Depends} Description: Xilinx Intermediate Representation (XIR) for deep learning algorithms (runtime) @@ -53,10 +54,10 @@ Description: Xilinx Intermediate Represe This package provides the runtime environment for XIR. Package: libxir-dev -Architecture: amd64 arm64 armhf +Architecture: amd64 arm64 armhf armel Section: libdevel Multi-Arch: same -Depends: ${misc:Depends}, ${shlibs:Depends}, libxir1 (= ${binary:Version}) +Depends: ${misc:Depends}, ${shlibs:Depends}, libxir2 (= ${binary:Version}) Description: Xilinx Intermediate Representation (XIR) for deep learning algorithms (develop) Xilinx Intermediate Representation (XIR) is a graph based intermediate representation of the AI algorithms which is well @@ -91,9 +92,9 @@ Description: Xilinx Intermediate Represe This package provides the development environment for XIR. Package: libxir-utils -Architecture: amd64 arm64 armhf +Architecture: amd64 arm64 armhf armel Section: utils -Depends: ${misc:Depends}, ${shlibs:Depends}, libxir1 (= ${binary:Version}) +Depends: ${misc:Depends}, ${shlibs:Depends}, libxir2 (= ${binary:Version}) Description: Xilinx Intermediate Representation (XIR) for deep learning algorithms (utils) Xilinx Intermediate Representation (XIR) is a graph based intermediate representation of the AI algorithms which is well diff -pruN 1.4-2/debian/libxir1.install 2.5-1/debian/libxir1.install --- 1.4-2/debian/libxir1.install 2021-10-21 02:43:54.000000000 +0000 +++ 2.5-1/debian/libxir1.install 1970-01-01 00:00:00.000000000 +0000 @@ -1 +0,0 @@ -usr/lib/*/*.so.* \ No newline at end of file diff -pruN 1.4-2/debian/libxir2.install 2.5-1/debian/libxir2.install --- 1.4-2/debian/libxir2.install 1970-01-01 00:00:00.000000000 +0000 +++ 2.5-1/debian/libxir2.install 2022-08-03 07:38:14.000000000 +0000 @@ -0,0 +1 @@ +usr/lib/*/*.so.* \ No newline at end of file diff -pruN 1.4-2/debian/patches/0001-Enable-multiarch-for-the-library.patch 2.5-1/debian/patches/0001-Enable-multiarch-for-the-library.patch --- 1.4-2/debian/patches/0001-Enable-multiarch-for-the-library.patch 2021-10-21 02:43:54.000000000 +0000 +++ 2.5-1/debian/patches/0001-Enable-multiarch-for-the-library.patch 2022-08-03 07:38:14.000000000 +0000 @@ -1,6 +1,7 @@ +From 1411fb040e9c4b500fc7b04665234a248a94cfd9 Mon Sep 17 00:00:00 2001 From: Punit Agrawal Date: Wed, 7 Apr 2021 14:42:50 +0900 -Subject: Enable multiarch for the library +Subject: [PATCH 1/5] Enable multiarch for the library The library install path is hardcoded which gets in the way of enabling multiarch. @@ -9,19 +10,24 @@ Fix this by respecting the standard CMAK library install location. This variable gets used during package building to install the package to multiarch aware location. --- - src/xir/CMakeLists.txt | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) + src/xir/CMakeLists.txt | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/xir/CMakeLists.txt b/src/xir/CMakeLists.txt -index 9f0c9ed..68af3f3 100644 +index 8b77297..ebaafe3 100644 --- a/src/xir/CMakeLists.txt +++ b/src/xir/CMakeLists.txt -@@ -63,7 +63,7 @@ install( +@@ -85,8 +85,8 @@ install( TARGETS ${PROJECT_NAME} EXPORT ${PROJECT_NAME}-targets RUNTIME DESTINATION bin +- ARCHIVE DESTINATION lib - LIBRARY DESTINATION lib) ++ ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) - foreach(PUBLIC_HEADER - xir/xir.h - xir/op/op.hpp + foreach( + PUBLIC_HEADER + xir/XirExport.hpp +-- +2.36.0 + diff -pruN 1.4-2/debian/patches/0002-Drop-dependency-on-source-repository-hash.patch 2.5-1/debian/patches/0002-Drop-dependency-on-source-repository-hash.patch --- 1.4-2/debian/patches/0002-Drop-dependency-on-source-repository-hash.patch 2021-10-21 02:43:54.000000000 +0000 +++ 2.5-1/debian/patches/0002-Drop-dependency-on-source-repository-hash.patch 2022-08-03 07:38:14.000000000 +0000 @@ -1,6 +1,7 @@ +From 3ae053f77bced8c6bc0e382809f7c587e38ac257 Mon Sep 17 00:00:00 2001 From: Punit Agrawal Date: Wed, 7 Apr 2021 15:31:18 +0900 -Subject: Drop dependency on source repository hash +Subject: [PATCH 2/5] Drop dependency on source repository hash xir uses a hash of the top commit of the repository as library id. As this id depends on the repository / branch it is not stable; a version @@ -14,10 +15,10 @@ library interface. 2 files changed, 12 deletions(-) diff --git a/include/xir/util/tool_function.hpp b/include/xir/util/tool_function.hpp -index 3ef7d18..7a7c7d3 100644 +index dcb384a..76d72f9 100644 --- a/include/xir/util/tool_function.hpp +++ b/include/xir/util/tool_function.hpp -@@ -45,13 +45,6 @@ const std::string get_md5_of_file(const std::string& filepath); +@@ -57,13 +57,6 @@ XIR_DLLESPEC const std::string get_md5_of_file(const std::string& filepath); */ const std::string get_lib_name(); @@ -32,10 +33,10 @@ index 3ef7d18..7a7c7d3 100644 namespace th { diff --git a/src/xir/util/tool_function.cpp b/src/xir/util/tool_function.cpp -index 61f456b..2ad67b2 100644 +index 61fc7aa..685039f 100644 --- a/src/xir/util/tool_function.cpp +++ b/src/xir/util/tool_function.cpp -@@ -67,11 +67,6 @@ const std::string get_lib_name() { +@@ -85,11 +85,6 @@ const std::string get_lib_name() { return ret; } @@ -47,3 +48,6 @@ index 61f456b..2ad67b2 100644 // name related void add_prefix_helper(std::string& name, const std::string& prefix) { std::string prefix_inst = HEAD_DELIMITER + prefix + TAIL_DELIMITER; +-- +2.36.0 + diff -pruN 1.4-2/debian/patches/0003-Install-cmake-files-in-multiarch-aware-locations.patch 2.5-1/debian/patches/0003-Install-cmake-files-in-multiarch-aware-locations.patch --- 1.4-2/debian/patches/0003-Install-cmake-files-in-multiarch-aware-locations.patch 2021-10-21 02:43:54.000000000 +0000 +++ 2.5-1/debian/patches/0003-Install-cmake-files-in-multiarch-aware-locations.patch 2022-08-03 07:38:14.000000000 +0000 @@ -1,6 +1,7 @@ +From df8da856a14ef9688200ae4e90203d585a587efa Mon Sep 17 00:00:00 2001 From: Punit Agrawal Date: Thu, 8 Apr 2021 11:59:20 +0900 -Subject: Install cmake files in multiarch aware locations +Subject: [PATCH 3/5] Install cmake files in multiarch aware locations --- CMakeLists.txt | 4 ++-- @@ -8,34 +9,37 @@ Subject: Install cmake files in multiarc 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt -index d6f80ad..5ba5ed1 100644 +index 551a9e5..4d475d3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt -@@ -35,7 +35,7 @@ add_subdirectory(src) +@@ -42,7 +42,7 @@ add_subdirectory(src) add_subdirectory(tools) file(RELATIVE_PATH REL_INCLUDE_DIR -- ${CMAKE_INSTALL_PREFIX}/share/cmake/${PROJECT_NAME} -+ ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME} - ${CMAKE_INSTALL_PREFIX}/include) +- ${CMAKE_INSTALL_PREFIX}/share/cmake/${PROJECT_NAME} ++ ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME} + ${CMAKE_INSTALL_PREFIX}/include) set(CONF_INCLUDE_DIRS "\${${PROJECT_NAME}_CMAKE_DIR}/${REL_INCLUDE_DIR}") configure_file(cmake/config.cmake.in -@@ -50,7 +50,7 @@ write_basic_package_version_file( - install(FILES - "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}-config.cmake" - "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}-config-version.cmake" -- DESTINATION share/cmake/${PROJECT_NAME}) -+ DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}) +@@ -55,7 +55,7 @@ write_basic_package_version_file( + + install(FILES "${CMAKE_BINARY_DIR}/${PROJECT_NAME}-config.cmake" + "${CMAKE_BINARY_DIR}/${PROJECT_NAME}-config-version.cmake" +- DESTINATION share/cmake/${PROJECT_NAME}) ++ DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}) if(BUILD_TEST) add_subdirectory(test) diff --git a/src/xir/CMakeLists.txt b/src/xir/CMakeLists.txt -index 68af3f3..03a1938 100644 +index ebaafe3..9d4509d 100644 --- a/src/xir/CMakeLists.txt +++ b/src/xir/CMakeLists.txt -@@ -85,4 +85,4 @@ endforeach() +@@ -109,4 +109,4 @@ endforeach() install( EXPORT ${PROJECT_NAME}-targets NAMESPACE ${PROJECT_NAME}:: - DESTINATION share/cmake/${PROJECT_NAME}) + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}) +-- +2.36.0 + diff -pruN 1.4-2/debian/patches/0004-cmake-vitis_version.c.in-Remove-the-build-date-from-.patch 2.5-1/debian/patches/0004-cmake-vitis_version.c.in-Remove-the-build-date-from-.patch --- 1.4-2/debian/patches/0004-cmake-vitis_version.c.in-Remove-the-build-date-from-.patch 1970-01-01 00:00:00.000000000 +0000 +++ 2.5-1/debian/patches/0004-cmake-vitis_version.c.in-Remove-the-build-date-from-.patch 2022-08-03 07:38:14.000000000 +0000 @@ -0,0 +1,27 @@ +From 76965a3698355f58d4de5f8f8efab5bfd739463c Mon Sep 17 00:00:00 2001 +From: Vagrant Cascadian +Date: Sun, 13 Feb 2022 03:48:45 +0000 +Subject: [PATCH 4/5] cmake/vitis_version.c.in: Remove the build date from the + embedded version. + +https://reproducible-builds.org/docs/timestamps/ +--- + cmake/vitis_version.c.in | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/cmake/vitis_version.c.in b/cmake/vitis_version.c.in +index a63825b..2748170 100644 +--- a/cmake/vitis_version.c.in ++++ b/cmake/vitis_version.c.in +@@ -17,7 +17,7 @@ const char* xilinx_@PROJECT_NAME@_version() { + return "@PROJECT_NAME@-@GIT_VERSION@@VERSION_SUFFIX@"; + } + +-static char @PROJECT_NAME@_version[] = "Xilinx @PROJECT_NAME@ Version: @PROJECT_NAME@-@GIT_VERSION@@VERSION_SUFFIX@ @BUILD_DATE@"; ++static char @PROJECT_NAME@_version[] = "Xilinx @PROJECT_NAME@ Version: @PROJECT_NAME@-@GIT_VERSION@@VERSION_SUFFIX@"; + const char* xilinx_version() { + return @PROJECT_NAME@_version; + } +-- +2.36.0 + diff -pruN 1.4-2/debian/patches/0004-Convert-loop-variables-to-reference-type.patch 2.5-1/debian/patches/0004-Convert-loop-variables-to-reference-type.patch --- 1.4-2/debian/patches/0004-Convert-loop-variables-to-reference-type.patch 2021-10-21 02:43:54.000000000 +0000 +++ 2.5-1/debian/patches/0004-Convert-loop-variables-to-reference-type.patch 1970-01-01 00:00:00.000000000 +0000 @@ -1,53 +0,0 @@ -From: Punit Agrawal -Date: Tue, 19 Oct 2021 10:01:34 +0900 -Subject: Convert loop variables to reference type -MIME-Version: 1.0 -Content-Type: text/plain; charset="utf-8" -Content-Transfer-Encoding: 8bit - -Recent versions of the compiler enable "-Werror=range-loop-construct" -by default which leads to errors such as - - - /src/xir/graph/elf2xir.cpp:403:19: error: loop variable ‘x’ creates a copy from type ‘const std::__cxx11::basic_string’ [-Werror=range-loop-construct] - 403 | for (const auto x : v) { - | ^ - /src/xir/graph/elf2xir.cpp:403:19: note: use reference type to prevent copying - 403 | for (const auto x : v) { - | ^ - | & - -Fix a couple of instances of the same problem by converting the loop -variable to reference type. - -Closes: #996583 ---- - src/xir/graph/elf2xir.cpp | 2 +- - tools/cmd_graph.cpp | 2 +- - 2 files changed, 2 insertions(+), 2 deletions(-) - -diff --git a/src/xir/graph/elf2xir.cpp b/src/xir/graph/elf2xir.cpp -index 7ea741f..b4bd084 100644 ---- a/src/xir/graph/elf2xir.cpp -+++ b/src/xir/graph/elf2xir.cpp -@@ -400,7 +400,7 @@ static inline std::ostream& operator<<(std::ostream& out, - const std::vector& v) { - int c = 0; - out << "["; -- for (const auto x : v) { -+ for (const auto &x : v) { - if (c++ != 0) { - out << ","; - } -diff --git a/tools/cmd_graph.cpp b/tools/cmd_graph.cpp -index 831fe8f..e9e1822 100644 ---- a/tools/cmd_graph.cpp -+++ b/tools/cmd_graph.cpp -@@ -109,7 +109,7 @@ template - std::enable_if_t && is_cout_able_v, std::ostream&> - operator<<(std::ostream& out, const std::map& v) { - out << "{"; -- for (const auto x : v) { -+ for (const auto &x : v) { - out << "\n\t\"" << x.first << "\" = " << x.second; - } - out << "\n}"; diff -pruN 1.4-2/debian/patches/0005-Don-t-build-with-Werror.patch 2.5-1/debian/patches/0005-Don-t-build-with-Werror.patch --- 1.4-2/debian/patches/0005-Don-t-build-with-Werror.patch 1970-01-01 00:00:00.000000000 +0000 +++ 2.5-1/debian/patches/0005-Don-t-build-with-Werror.patch 2022-08-03 07:38:14.000000000 +0000 @@ -0,0 +1,40 @@ +From fcaf3e22964981b838a3adc894449a398ed9e877 Mon Sep 17 00:00:00 2001 +From: Nobuhiro Iwamatsu +Date: Mon, 6 Jun 2022 18:12:54 +0900 +Subject: [PATCH 5/5] Don't build with -Werror + +Description: Don't build with -Werror + Thie frequently breaks when new gcc or OpenSSL versions bring new +warnings. +Author: Adrian Bunk +Bug-Debian: https://bugs.debian.org/1006582 + +Signed-off-by: Nobuhiro Iwamatsu +--- + cmake/VitisCommon.cmake | 10 +++++----- + 1 file changed, 5 insertions(+), 5 deletions(-) + +diff --git a/cmake/VitisCommon.cmake b/cmake/VitisCommon.cmake +index 4e78820..3fba250 100644 +--- a/cmake/VitisCommon.cmake ++++ b/cmake/VitisCommon.cmake +@@ -19,11 +19,11 @@ set (CMAKE_C_STANDARD 99) + if (MSVC) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Zc:__cplusplus") + else(MSVC) +- set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wall -Werror -ggdb -O0 -fno-inline") +- set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3 -Wall -Werror") +- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Werror") +- set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Werror") +- set(CMAKE_EXE "${CMAKE_C_FLAGS} -Wall -Werror") ++ set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wall -Werror -O0 -fno-inline") ++ set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3 -Wall") ++ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall") ++ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall") ++ set(CMAKE_EXE "${CMAKE_C_FLAGS} -Wall") + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--no-undefined") + set(CMAKE_MACOSX_RPATH 1) + endif(MSVC) +-- +2.36.0 + diff -pruN 1.4-2/debian/patches/Fix-FTBFS-with-Werror-range-loop-construct-Closes-99.patch 2.5-1/debian/patches/Fix-FTBFS-with-Werror-range-loop-construct-Closes-99.patch --- 1.4-2/debian/patches/Fix-FTBFS-with-Werror-range-loop-construct-Closes-99.patch 2021-10-21 02:43:54.000000000 +0000 +++ 2.5-1/debian/patches/Fix-FTBFS-with-Werror-range-loop-construct-Closes-99.patch 1970-01-01 00:00:00.000000000 +0000 @@ -1,41 +0,0 @@ -From 488f0aaba5bb0b71feee801d82000d9e6a20144a Mon Sep 17 00:00:00 2001 -From: Nobuhiro Iwamatsu -Date: Tue, 19 Oct 2021 10:44:56 +0900 -Subject: [PATCH] Fix FTBFS with '-Werror=range-loop-construct' (Closes: - #996583) - -Signed-off-by: Nobuhiro Iwamatsu ---- - src/xir/graph/elf2xir.cpp | 2 +- - tools/cmd_graph.cpp | 2 +- - 2 files changed, 2 insertions(+), 2 deletions(-) - -diff --git a/src/xir/graph/elf2xir.cpp b/src/xir/graph/elf2xir.cpp -index 7ea741f..b4bd084 100644 ---- a/src/xir/graph/elf2xir.cpp -+++ b/src/xir/graph/elf2xir.cpp -@@ -400,7 +400,7 @@ static inline std::ostream& operator<<(std::ostream& out, - const std::vector& v) { - int c = 0; - out << "["; -- for (const auto x : v) { -+ for (const auto &x : v) { - if (c++ != 0) { - out << ","; - } -diff --git a/tools/cmd_graph.cpp b/tools/cmd_graph.cpp -index 831fe8f..e9e1822 100644 ---- a/tools/cmd_graph.cpp -+++ b/tools/cmd_graph.cpp -@@ -109,7 +109,7 @@ template - std::enable_if_t && is_cout_able_v, std::ostream&> - operator<<(std::ostream& out, const std::map& v) { - out << "{"; -- for (const auto x : v) { -+ for (const auto &x : v) { - out << "\n\t\"" << x.first << "\" = " << x.second; - } - out << "\n}"; --- -2.33.0 - diff -pruN 1.4-2/debian/patches/series 2.5-1/debian/patches/series --- 1.4-2/debian/patches/series 2021-10-21 02:43:54.000000000 +0000 +++ 2.5-1/debian/patches/series 2022-08-03 07:38:14.000000000 +0000 @@ -1,4 +1,5 @@ 0001-Enable-multiarch-for-the-library.patch 0002-Drop-dependency-on-source-repository-hash.patch 0003-Install-cmake-files-in-multiarch-aware-locations.patch -Fix-FTBFS-with-Werror-range-loop-construct-Closes-99.patch +0004-cmake-vitis_version.c.in-Remove-the-build-date-from-.patch +0005-Don-t-build-with-Werror.patch diff -pruN 1.4-2/debian/rules 2.5-1/debian/rules --- 1.4-2/debian/rules 2021-10-21 02:43:54.000000000 +0000 +++ 2.5-1/debian/rules 2022-08-03 07:38:14.000000000 +0000 @@ -3,9 +3,10 @@ export DH_VERBOSE = 1 export DEB_BUILD_MAINT_OPTIONS = hardening=+all export DEB_CFLAGS_MAINT_APPEND = -Wall -pedantic -export DEB_LDFLAGS_MAINT_APPEND = -Wl,--as-needed -CONFIGURE_ARGS = -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_FLAGS_DEBUG="" +CONFIGURE_ARGS = -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_BUILD_RPATH_USE_ORIGIN=ON \ + -DCMAKE_CXX_FLAGS_DEBUG="-std=c++17" %: dh $@ diff -pruN 1.4-2/debian/upstream/metadata 2.5-1/debian/upstream/metadata --- 1.4-2/debian/upstream/metadata 1970-01-01 00:00:00.000000000 +0000 +++ 2.5-1/debian/upstream/metadata 2022-08-03 07:38:14.000000000 +0000 @@ -0,0 +1,5 @@ +--- +Bug-Database: https://github.com/Xilinx/Vitis-AI/issues +Bug-Submit: https://github.com/Xilinx/Vitis-AI/issues/new +Repository: https://github.com/Xilinx/Vitis-AI.git +Repository-Browse: https://github.com/Xilinx/Vitis-AI diff -pruN 1.4-2/doc/doxygen/CMakeLists.txt 2.5-1/doc/doxygen/CMakeLists.txt --- 1.4-2/doc/doxygen/CMakeLists.txt 2021-07-27 05:41:52.000000000 +0000 +++ 2.5-1/doc/doxygen/CMakeLists.txt 2022-06-15 07:33:00.000000000 +0000 @@ -16,7 +16,7 @@ include(FindDoxygen) # doxygen setting -set(DOXYGEN_CONFIG_DIR ${CMAKE_SOURCE_DIR}/doc/doxygen) +set(DOXYGEN_CONFIG_DIR ${PROJECT_SOURCE_DIR}/doc/doxygen) configure_file(${DOXYGEN_CONFIG_DIR}/Doxyfile.in ${CMAKE_BINARY_DIR}/Doxyfile @ONLY) add_custom_target(doc ALL COMMAND ${DOXYGEN_EXECUTABLE} ${CMAKE_BINARY_DIR}/Doxyfile @@ -27,5 +27,5 @@ add_custom_target(doc ALL WORKING_DIRECTORY ${CMAKE_BINARY_DIR} COMMENT "Generating Documentation with Doxygen" VERBATIM ) -install(FILES ${CMAKE_BINARY_DIR}/c-doc/latex/refman.pdf DESTINATION ${CMAKE_SOURCE_DIR}/doc +install(FILES ${CMAKE_BINARY_DIR}/c-doc/latex/refman.pdf DESTINATION ${PROJECT_SOURCE_DIR}/doc RENAME xir-reference-manual.pdf) diff -pruN 1.4-2/doc/doxygen/Doxyfile.in 2.5-1/doc/doxygen/Doxyfile.in --- 1.4-2/doc/doxygen/Doxyfile.in 2021-07-27 05:41:52.000000000 +0000 +++ 2.5-1/doc/doxygen/Doxyfile.in 2022-06-15 07:33:00.000000000 +0000 @@ -13,13 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -INPUT = @CMAKE_SOURCE_DIR@/README.md \ - @CMAKE_SOURCE_DIR@/include/xir/op/op_def.hpp \ - @CMAKE_SOURCE_DIR@/include/xir/op/op.hpp \ - @CMAKE_SOURCE_DIR@/include/xir/tensor/tensor.hpp \ - @CMAKE_SOURCE_DIR@/include/xir/graph/graph.hpp \ - @CMAKE_SOURCE_DIR@/include/xir/graph/subgraph.hpp \ - @CMAKE_SOURCE_DIR@/include/xir/attrs/attrs.hpp \ +INPUT = @PROJECT_SOURCE_DIR@/README.md \ + @PROJECT_SOURCE_DIR@/include/xir/op/op_def.hpp \ + @PROJECT_SOURCE_DIR@/include/xir/op/op.hpp \ + @PROJECT_SOURCE_DIR@/include/xir/tensor/tensor.hpp \ + @PROJECT_SOURCE_DIR@/include/xir/graph/graph.hpp \ + @PROJECT_SOURCE_DIR@/include/xir/graph/subgraph.hpp \ + @PROJECT_SOURCE_DIR@/include/xir/attrs/attrs.hpp \ ENABLE_PREPROCESSING = YES @@ -36,10 +36,10 @@ HTML_OUTPUT = . OUTPUT_DIRECTORY = @CMAKE_BINARY_DIR@/c-doc GENERATE_LATEX = YES GENERATE_HTML = YES -LATEX_HEADER = @CMAKE_SOURCE_DIR@/doc/doxygen/header.tex -LATEX_FOOTER = @CMAKE_SOURCE_DIR@/doc/doxygen/footer.tex +LATEX_HEADER = @PROJECT_SOURCE_DIR@/doc/doxygen/header.tex +LATEX_FOOTER = @PROJECT_SOURCE_DIR@/doc/doxygen/footer.tex #ENABLED_SECTIONS = YES ALPHABETICAL_INDEX = NO #HTML_DYNAMIC_MENUS=NO -USE_MDFILE_AS_MAINPAGE = @CMAKE_SOURCE_DIR@/README.md +USE_MDFILE_AS_MAINPAGE = @PROJECT_SOURCE_DIR@/README.md diff -pruN 1.4-2/include/xir/attrs/attr_def.hpp 2.5-1/include/xir/attrs/attr_def.hpp --- 1.4-2/include/xir/attrs/attr_def.hpp 2021-07-27 05:41:52.000000000 +0000 +++ 2.5-1/include/xir/attrs/attr_def.hpp 2022-06-15 07:33:00.000000000 +0000 @@ -67,7 +67,7 @@ extern std::type_index TYPE_INDEX_MAP_ST *@brief attribute definition *This struct defines an attribute, like 'kernel_w' for conv2d */ -struct AttrDef { +struct XIR_DLLESPEC AttrDef { /** * @brief Element Occurence Specifier */ @@ -78,19 +78,23 @@ struct AttrDef { OPTIONAL, NUM }; - + /// MSVC NOTE: member variable cannot be const, because + /// vector in OpDef requires it is copiable. // when + /// exported with XIR_DLLESPEC, a special default copy constructor + /// is created, which is deleted. + // /// Name of the op attribute - const std::string name; + std::string name; /// Data type - const std::type_index data_type; + std::type_index data_type; /// Occurence type - const OccurenceType occur_type; + OccurenceType occur_type; /// List size for validation, 0 for variable length - const std::uint32_t list_length; + std::uint32_t list_length; /// Some comments - const std::string annotation; + std::string annotation; /// Default value of the attribute - const xir::any default_value; + xir::any default_value; }; template @@ -125,7 +129,7 @@ const AttrDef build_required_attr(const const std::string& annotation) { UNI_LOG_CHECK(occur_type == AttrDef::OccurenceType::REQUIRED, XIR_UNEXPECTED_VALUE) - << "REQUIRED item does not need to have a default value"; + << "REQUIRED item does not need to have a default value"; return AttrDef{name, std::type_index{typeid(T)}, AttrDef::OccurenceType::REQUIRED, @@ -142,7 +146,7 @@ const AttrDef build_optional_attr(const const T& default_value) { UNI_LOG_CHECK(occur_type == AttrDef::OccurenceType::OPTIONAL, XIR_UNEXPECTED_VALUE) - << "OPTIONAL item needs to have a default value"; + << "OPTIONAL item needs to have a default value"; return AttrDef{name, std::type_index{typeid(T)}, AttrDef::OccurenceType::OPTIONAL, @@ -155,26 +159,19 @@ const AttrDef build_optional_attr(const // for scale type template struct AttrDefBuilder< - T, typename std::enable_if::value>::type> { + T, typename std::enable_if::value>::type> { static const AttrDef build(std::string name, AttrDef::OccurenceType occur_type, std::string annotation) { - return build_required_attr(name, - occur_type, - 1, - annotation); + return build_required_attr(name, occur_type, 1, annotation); } static const AttrDef build(std::string name, AttrDef::OccurenceType occur_type, - std::string annotation, - const T& default_value) { - return build_optional_attr(name, - occur_type, - 1, - annotation, + std::string annotation, const T& default_value) { + return build_optional_attr(name, occur_type, 1, annotation, default_value); } }; @@ -182,32 +179,25 @@ struct AttrDefBuilder< // for map type template struct AttrDefBuilder< - T, typename std::enable_if< - is_std_map::value && - is_one_of::value && - is_one_of>::value>::type> { + T, typename std::enable_if< + is_std_map::value && + is_one_of::value && + is_one_of>::value>::type> { static const AttrDef build(std::string name, AttrDef::OccurenceType occur_type, std::string annotation) { - return build_required_attr(name, - occur_type, - 1, - annotation); + return build_required_attr(name, occur_type, 1, annotation); } static const AttrDef build(std::string name, AttrDef::OccurenceType occur_type, - std::string annotation, - const T& default_value) { - return build_optional_attr(name, - occur_type, - 1, - annotation, + std::string annotation, const T& default_value) { + return build_optional_attr(name, occur_type, 1, annotation, default_value); } }; @@ -215,30 +205,22 @@ struct AttrDefBuilder< // for vector type template struct AttrDefBuilder< - T, typename std::enable_if< - is_std_vector::value && - is_one_of::value>::type> { + T, typename std::enable_if< + is_std_vector::value && + is_one_of::value>::type> { static const AttrDef build(std::string name, AttrDef::OccurenceType occur_type, - std::uint32_t length, - std::string annotation) { - return build_required_attr(name, - occur_type, - length, - annotation); + std::uint32_t length, std::string annotation) { + return build_required_attr(name, occur_type, length, annotation); } static const AttrDef build(std::string name, AttrDef::OccurenceType occur_type, - std::uint32_t length, - std::string annotation, + std::uint32_t length, std::string annotation, const T& default_value) { - return build_optional_attr(name, - occur_type, - length, - annotation, + return build_optional_attr(name, occur_type, length, annotation, default_value); } }; diff -pruN 1.4-2/include/xir/attrs/attrs.hpp 2.5-1/include/xir/attrs/attrs.hpp --- 1.4-2/include/xir/attrs/attrs.hpp 2021-07-27 05:41:52.000000000 +0000 +++ 2.5-1/include/xir/attrs/attrs.hpp 2022-06-15 07:33:00.000000000 +0000 @@ -20,6 +20,7 @@ #include #include "UniLog/UniLog.hpp" +#include "xir/XirExport.hpp" #include "xir/util/any.hpp" namespace xir { @@ -28,7 +29,7 @@ namespace xir { * @brief Attrs is a interface class for users to access a set of attributes. * */ -class Attrs { +class XIR_DLLESPEC Attrs { public: Attrs() = default; Attrs(const Attrs& other) = default; diff -pruN 1.4-2/include/xir/graph/graph.hpp 2.5-1/include/xir/graph/graph.hpp --- 1.4-2/include/xir/graph/graph.hpp 2021-07-27 05:41:52.000000000 +0000 +++ 2.5-1/include/xir/graph/graph.hpp 2022-06-15 07:33:00.000000000 +0000 @@ -21,6 +21,7 @@ #include #include +#include "xir/XirExport.hpp" #include "xir/graph/graph_template.hpp" #include "xir/graph/subgraph.hpp" #include "xir/op/op.hpp" @@ -45,7 +46,7 @@ namespace xir { * will support LSTM in another way rather than adding an edge directly. * Maintaining a graph with rings is much harder than acyclic graph. */ -class Graph { +class XIR_DLLESPEC Graph { public: /** * @brief Static function to create a graph with a name. diff -pruN 1.4-2/include/xir/graph/graph_template.hpp 2.5-1/include/xir/graph/graph_template.hpp --- 1.4-2/include/xir/graph/graph_template.hpp 2021-07-27 05:41:52.000000000 +0000 +++ 2.5-1/include/xir/graph/graph_template.hpp 2022-06-15 07:33:00.000000000 +0000 @@ -22,6 +22,7 @@ #include #include #include +#include "xir/XirExport.hpp" #include "xir/op/op.hpp" namespace xir { @@ -48,7 +49,7 @@ class OpTemplate { virtual ~OpTemplate() = default; }; -class GraphTemplate { +class XIR_DLLESPEC GraphTemplate { public: static std::unique_ptr create(std::string name); diff -pruN 1.4-2/include/xir/graph/subgraph.hpp 2.5-1/include/xir/graph/subgraph.hpp --- 1.4-2/include/xir/graph/subgraph.hpp 2021-07-27 05:41:52.000000000 +0000 +++ 2.5-1/include/xir/graph/subgraph.hpp 2022-06-15 07:33:00.000000000 +0000 @@ -22,9 +22,11 @@ #include #include +#include "xir/XirExport.hpp" #include "xir/graph/graph_template.hpp" #include "xir/op/op.hpp" #include "xir/util/any.hpp" +#include "xir/util/tool_function.hpp" namespace xir { @@ -35,7 +37,7 @@ class Graph; * * @brief A class for subgraph */ -class Subgraph { +class XIR_DLLESPEC Subgraph { public: /** * @brief Get the name of subgraph. @@ -111,6 +113,24 @@ class Subgraph { virtual const std::set get_input_tensors() const = 0; /** + * @brief Get all the sorted input tensors produced by other subgraph. + * + * @return A vector of raw pointer to the input tensors. + */ + virtual std::vector get_sorted_input_tensors( + const std::function& compare = + TensorLexicographicalOrder) = 0; + + /** + * @brief Get all the sorted input tensors produced by other subgraph. + * + * @return A vector of raw pointer to the input tensors. + */ + virtual const std::vector get_sorted_input_tensors( + const std::function& compare = + TensorLexicographicalOrder) const = 0; + + /** * @brief Get all the tensors output to other subgraphs or dump out in current * subgraph. * @@ -135,6 +155,34 @@ class Subgraph { virtual const std::set get_output_tensors() const = 0; /** + * @brief Get all the sorted tensors output to other subgraphs or dump out in + * current subgraph. + * + * @details There are two parts inside the output tensors. First, the tensor + * is passed to another subgraph as an input tensor; second, the tensor is + * dump out in the current subgraph, for instance an op without fanout. + * + * @return A vector of raw pointer to the output tensors. + */ + virtual std::vector get_sorted_output_tensors( + const std::function& compare = + TensorLexicographicalOrder) = 0; + + /** + * @brief Get all the sorted tensors output to other subgraphs or dump out in + * current subgraph. + * + * @details There are two parts inside the output tensors. First, the tensor + * is passed to another subgraph as an input tensor; second, the tensor is + * dump out in the current subgraph, for instance an op without fanout. + * + * @return A vector of raw pointer to the output tensors. + */ + virtual const std::vector get_sorted_output_tensors( + const std::function& compare = + TensorLexicographicalOrder) const = 0; + + /** * @brief Check the existence of the op indicated by name. * * @param op_name The name of the op. diff -pruN 1.4-2/include/xir/op/op_def.hpp 2.5-1/include/xir/op/op_def.hpp --- 1.4-2/include/xir/op/op_def.hpp 2021-07-27 05:41:52.000000000 +0000 +++ 2.5-1/include/xir/op/op_def.hpp 2022-06-15 07:33:00.000000000 +0000 @@ -40,7 +40,7 @@ namespace xir { *@brief Op argument definition *This struct defines an input argument of an op. */ -struct OpArgDef { +struct XIR_DLLESPEC OpArgDef { /** * @brief Element Occurence Specifier */ @@ -55,15 +55,20 @@ struct OpArgDef { REQUIRED_AND_REPEATED, NUM }; + /// MSVC NOTE: member variable cannot be const, because + /// vector in OpDef requires it is copiable. // when + /// exported with XIR_DLLESPEC, a special default copy constructor + /// is created, which is deleted. + // /// Name of the op argument - const std::string name; + std::string name; /// Occurence type - const OccurenceType occur_type; + OccurenceType occur_type; /// DataType - const xir::DataType::Type data_type; + xir::DataType::Type data_type; /// Some comments - const std::string annotation; + std::string annotation; }; /* @@ -71,7 +76,7 @@ struct OpArgDef { *@brief Op definition *This class defines an op. */ -class OpDef { +class XIR_DLLESPEC OpDef { public: /// Create a definition of an op by name OpDef(const std::string& name); @@ -112,12 +117,14 @@ class OpDef { const std::string& annotation() const; private: - std::string name_{}; - std::vector input_args_{}; - std::vector attrs_{}; + // MSVC NOTE: when export with XIR_DLLESPEC, we must not have + // variable constructor syntax here, it causes `operator=()` deleted. + std::string name_; + std::vector input_args_; + std::vector attrs_; std::function shape_infer_; - std::vector> constraints_{}; - std::string annotation_{}; + std::vector> constraints_; + std::string annotation_; }; class OpDefFactory { diff -pruN 1.4-2/include/xir/op/op.hpp 2.5-1/include/xir/op/op.hpp --- 1.4-2/include/xir/op/op.hpp 2021-07-27 05:41:52.000000000 +0000 +++ 2.5-1/include/xir/op/op.hpp 2022-06-15 07:33:00.000000000 +0000 @@ -21,6 +21,7 @@ #include #include +#include "xir/XirExport.hpp" namespace xir { @@ -32,7 +33,7 @@ class Graph; * * This class defines the basic XIR OP Interface. */ -class Op { +class XIR_DLLESPEC Op { public: /** * @brief Get name of the OP diff -pruN 1.4-2/include/xir/tensor/tensor.hpp 2.5-1/include/xir/tensor/tensor.hpp --- 1.4-2/include/xir/tensor/tensor.hpp 2021-07-27 05:41:52.000000000 +0000 +++ 2.5-1/include/xir/tensor/tensor.hpp 2022-06-15 07:33:00.000000000 +0000 @@ -19,6 +19,7 @@ #include #include +#include "xir/XirExport.hpp" #include "xir/attrs/attrs.hpp" #include "xir/util/data_type.hpp" @@ -31,7 +32,7 @@ class Op; * * This class defines the basic XIR Tensor Interface. */ -class Tensor { +class XIR_DLLESPEC Tensor { public: /** * @brief Create a Tensor instance. diff -pruN 1.4-2/include/xir/util/any.hpp 2.5-1/include/xir/util/any.hpp --- 1.4-2/include/xir/util/any.hpp 2021-07-27 05:41:52.000000000 +0000 +++ 2.5-1/include/xir/util/any.hpp 2022-06-15 07:33:00.000000000 +0000 @@ -17,9 +17,6 @@ #pragma once #if __has_include() && __cplusplus > 201700 #include -#if !__cpp_lib_any > 0 -#error "must enable -std=c++17 to use std::any" -#endif namespace xir { using any = std::any; namespace stdx = ::std; diff -pruN 1.4-2/include/xir/util/data_type.hpp 2.5-1/include/xir/util/data_type.hpp --- 1.4-2/include/xir/util/data_type.hpp 2021-07-27 05:41:52.000000000 +0000 +++ 2.5-1/include/xir/util/data_type.hpp 2022-06-15 07:33:00.000000000 +0000 @@ -18,10 +18,11 @@ #include #include #include +#include "xir/XirExport.hpp" namespace xir { -struct DataType { +struct XIR_DLLESPEC DataType { enum Type { INT, UINT, XINT, XUINT, FLOAT, UNKNOWN }; DataType(); @@ -31,8 +32,8 @@ struct DataType { const bool valid() const; const std::string to_string() const; - friend bool operator==(const DataType& lhs, const DataType& rhs); - friend bool operator!=(const DataType& lhs, const DataType& rhs); + XIR_DLLESPEC friend bool operator==(const DataType& lhs, const DataType& rhs); + XIR_DLLESPEC friend bool operator!=(const DataType& lhs, const DataType& rhs); Type type; std::int32_t bit_width; diff -pruN 1.4-2/include/xir/util/dynamic_load.hpp 2.5-1/include/xir/util/dynamic_load.hpp --- 1.4-2/include/xir/util/dynamic_load.hpp 1970-01-01 00:00:00.000000000 +0000 +++ 2.5-1/include/xir/util/dynamic_load.hpp 2022-06-15 07:33:00.000000000 +0000 @@ -0,0 +1,41 @@ +/* + * Copyright 2019 Xilinx Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#ifndef _WIN32 +#include +#else +#define WIN32_LEAN_AND_MEAN +#include +#undef OPTIONAL +#define RTLD_LAZY 0 +#define RTLD_GLOBAL 0 +#define RTLD_NOW 0 +#endif + +#ifdef _WIN32 +inline void* dlopen(const char* dllname, int) { return LoadLibrary(dllname); } + +inline void dlclose(void* handle) { FreeLibrary(HMODULE(handle)); } + +inline const char* dlerror() { return ""; } + +inline void* dlsym(void* handle, const char* symbol) { + return GetProcAddress(HMODULE(handle), symbol); +} + +#endif diff -pruN 1.4-2/include/xir/util/tool_function.hpp 2.5-1/include/xir/util/tool_function.hpp --- 1.4-2/include/xir/util/tool_function.hpp 2021-07-27 05:41:52.000000000 +0000 +++ 2.5-1/include/xir/util/tool_function.hpp 2022-06-15 07:33:00.000000000 +0000 @@ -18,25 +18,34 @@ #include #include #include +#include #include #include #include -#include -#include "UniLog/UniLog.hpp" -#include "xir/graph/subgraph.hpp" + #include "xir/op/op.hpp" -#include "xir/tensor/tensor.hpp" namespace xir { /** + * @brief Get the md5sum of a buff. + * + * @param buf The buffer base address. + * + * @param size The buffer's size, in bytes. + * + * @return A string contains the md5sum in hex format. + */ +XIR_DLLESPEC const std::string get_md5_of_buffer(const void* buf, size_t size); + +/** * @brief Get the md5sum of a file. * * @param filepath The path of the input file. * * @return A string contains the md5sum in hex format. */ -const std::string get_md5_of_file(const std::string& filepath); +XIR_DLLESPEC const std::string get_md5_of_file(const std::string& filepath); /** * @brief Get the xir-lib name. @@ -181,7 +190,7 @@ std::string add_suffix(const std::string * * @return The original name. */ -std::string remove_xfix(const std::string& name); +XIR_DLLESPEC std::string remove_xfix(const std::string& name); /** * @brief Extract all the prefix, suffix and the original name. @@ -205,4 +214,18 @@ std::vector extract_xfix(co */ float xround(const float& data, const std::string& round_mode = "STD_ROUND"); +void register_customized_operator_definition(const std::string& name, + const std::string& type); +std::vector get_float_vec_from_any(const xir::any& any); + +/** + * @brief Tensor lexicographical order sort function + * name. + * + * @param name The name. + * + * @return The original name. + */ +bool TensorLexicographicalOrder(Tensor* a, Tensor* b); + } // namespace xir diff -pruN 1.4-2/include/xir/XirExport.hpp 2.5-1/include/xir/XirExport.hpp --- 1.4-2/include/xir/XirExport.hpp 1970-01-01 00:00:00.000000000 +0000 +++ 2.5-1/include/xir/XirExport.hpp 2022-06-15 07:33:00.000000000 +0000 @@ -0,0 +1,26 @@ +/* + * Copyright 2019 Xilinx Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#if defined(_WIN32) + #ifdef XIR_EXPORT + #define XIR_DLLESPEC __declspec(dllexport) + #else + #define XIR_DLLESPEC __declspec(dllimport) + #endif +#else + #define XIR_DLLESPEC __attribute__((visibility("default"))) +#endif diff -pruN 1.4-2/include/xir/xir.h 2.5-1/include/xir/xir.h --- 1.4-2/include/xir/xir.h 2021-07-27 05:41:52.000000000 +0000 +++ 2.5-1/include/xir/xir.h 1970-01-01 00:00:00.000000000 +0000 @@ -1,375 +0,0 @@ -/* - * Copyright 2019 Xilinx Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once -#include -#include -#ifdef __cplusplus -extern "C" { -#endif -/** - * the obscure type for xir graph - */ -typedef void* xir_graph_t; - -/** - * the obscure type for xir subgraph - */ -typedef void* xir_subgraph_t; - -/** - * the obscure type for xir attrs - */ -typedef void* xir_attrs_t; - -/** - * the obscure type for xir Op - */ -typedef void* xir_op_t; - -/** - * the obscure type for xir tensor - */ -typedef void* xir_tensor_t; - -enum xir_tensor_data_type_t { - XIR_INT, - XIR_UINT, - XIR_XINT, - XIR_XUINT, - XIR_FLOAT, - UNKNOWN -}; - -/** - * @brief create a graph with a name. - * - * @param name Name of the created graph. - * - * @return An instance of graph. - */ - -xir_graph_t xir_graph_create(const char* name); - -/** - * @brief Deserializa a graph from a pb file. - * - * @param pb_fname The path of the pb file. - * - * @return A unique pointer to the graph object. - */ -xir_graph_t xir_graph_deserialize(const char* pb_fname); - -/** - * @brief destroy a graph - * - * @return return value is not in use yet. it is always zero. - */ -int xir_graph_destroy(xir_graph_t graph); -/** - * @brief Get name of the graph. - * - * @return Graph name. - */ -const char* xir_graph_get_name(xir_graph_t graph); - -/** - * @brief Get root subgraph of this graph. - * - * @return A pointer to root subgraph. - */ -xir_subgraph_t xir_graph_get_root_subgraph(xir_graph_t graph); - -/** - * @brief Add an operator to the graph. - * - * @details Note that the parameters have to satisfy conditions descrided in - * the parameter field, or the function will crash. Those conditions make sure - * that there will never be an invalid status during the graph modification. - * But that brings that you must build a graph carefully and add OPs in - * topologic order. - * - * @param name Name of the OP. It has to be different from every existed OP - * in the graph. - * - * @param type Type of the OP. It has to be registered into OpDefFactory. - * - * @param attrs Attributes of the OP. It has to contain all the - * required op attributes which are defined in OpDef. The ownership is - * taken away, the caller must not invoke xir_attrs_destroy() later on. - * - * @param input_ops_map Map of input operators where key is input argument - * name defined in OpDef and value is vector of input operator pointer. The - * number of the input operator has to be appropriate with the defination from - * OpDef. - * - * @param tensor The ownership is taken so that the caller must not - * invoke xir_tensor_destoy() later on. - * - * @return An instance of OP. - */ -struct xir_graph_input_ops_t { - const char* name; - xir_op_t* ops; - size_t num_of_ops; -}; - -xir_op_t xir_graph_add_op(xir_graph_t graph, // - const char* name, // - const char* type, // - xir_attrs_t attrs, // - struct xir_graph_input_ops_t* input_ops, // - size_t num_of_ops, // - xir_tensor_t tensor, // - xir_subgraph_t subgraph // -); - -/** - * @breif Create a new empty Attrs object, and return a unique pointer - * of the new object. - * - * @return A unique pointer to the created Attrs object. - */ - -xir_attrs_t xir_attrs_create(); -/** - * @brief destroy the xir attrs - */ -void xir_attrs_destroy(xir_attrs_t); - -// clang-format off -typedef const char *xir_string_t; -typedef struct { - int8_t * data; - size_t size; -} xir_bytes_t; -#define XIR_ATTRS_SUPPORTED_PRIMITIVE_TYPES(p) \ - p(bool, int, bool) \ - p(i8, int8_t, int8_t) \ - p(i16, int16_t, int16_t) \ - p(i32, int32_t, int32_t) \ - p(i64, int64_t, int64_t) \ - p(u8, uint8_t, uint8_t) \ - p(u16, uint16_t, uint16_t) \ - p(u32, uint32_t, uint32_t) \ - p(u64, uint64_t, uint64_t) \ - p(f32, float, float) \ - p(f64, double, double) \ - p(string, xir_string_t, std::string) \ - p(bytes, xir_bytes_t, std::vector) -// p(type_name, c_type, c++ type ) -#define DECLARE_XIR_ATTRS_GET(type_name, c_type, cxx_type) \ -c_type xir_attrs_get_##type_name (xir_attrs_t attrs, const char* name); -#define DECLARE_XIR_ATTRS_SET(type_name, c_type, cxx_type) \ -void xir_attrs_set_##type_name (xir_attrs_t attrs, const char* name, c_type value); -#define DECLARE_XIR_ATTRS_HAS_ATTRS(type_name, c_type, cxx_type) \ -int xir_attrs_has_##type_name (xir_attrs_t attrs, const char* name); -XIR_ATTRS_SUPPORTED_PRIMITIVE_TYPES (DECLARE_XIR_ATTRS_GET) -XIR_ATTRS_SUPPORTED_PRIMITIVE_TYPES (DECLARE_XIR_ATTRS_SET) -XIR_ATTRS_SUPPORTED_PRIMITIVE_TYPES (DECLARE_XIR_ATTRS_HAS_ATTRS) - -#define DECLARE_XIR_ATTRS_GET_VEC_SIZE(type_name, c_type, cxx_type) \ -size_t xir_attrs_get_vec_size_ ##type_name (xir_attrs_t attrs, const char* name); -#define DECLARE_XIR_ATTRS_GET_VEC(type_name, c_type, cxx_type) \ -c_type xir_attrs_get_vec_ ##type_name (xir_attrs_t attrs, const char* name, size_t idx); -#define DECLARE_XIR_ATTRS_SET_VEC(type_name, c_type, cxx_type) \ - void xir_attrs_set_vec_ ##type_name (xir_attrs_t attrs, const char* name, size_t idx, c_type value); -#define DECLARE_XIR_ATTRS_HAS_VEC(type_name, c_type, cxx_type) \ -int xir_attrs_has_vec_ ##type_name (xir_attrs_t attrs, const char* name); - -XIR_ATTRS_SUPPORTED_PRIMITIVE_TYPES(DECLARE_XIR_ATTRS_GET_VEC_SIZE) -XIR_ATTRS_SUPPORTED_PRIMITIVE_TYPES(DECLARE_XIR_ATTRS_GET_VEC) -XIR_ATTRS_SUPPORTED_PRIMITIVE_TYPES(DECLARE_XIR_ATTRS_SET_VEC) -XIR_ATTRS_SUPPORTED_PRIMITIVE_TYPES(DECLARE_XIR_ATTRS_HAS_VEC) - -#define DECLARE_XIR_ATTRS_GET_MAP_SIZE(type_name, c_type, cxx_type) \ -size_t xir_attrs_get_map_size_ ##type_name (xir_attrs_t attrs, const char* name); -#define DECLARE_XIR_ATTRS_GET_MAP_KEYS(type_name, c_type, cxx_type) \ -void xir_attrs_get_map_keys_ ##type_name (xir_attrs_t attrs, const char* name, const char *keys[]); -#define DECLARE_XIR_ATTRS_GET_MAP(type_name, c_type, cxx_type) \ -c_type xir_attrs_get_map_ ##type_name (xir_attrs_t attrs, const char* name, const char * key); -#define DECLARE_XIR_ATTRS_SET_MAP(type_name, c_type, cxx_type) \ - void xir_attrs_set_map_ ##type_name (xir_attrs_t attrs, const char* name, const char * key, c_type value); -#define DECLARE_XIR_ATTRS_HAS_MAP(type_name, c_type, cxx_type) \ -int xir_attrs_has_map_ ##type_name (xir_attrs_t attrs, const char* name); - -XIR_ATTRS_SUPPORTED_PRIMITIVE_TYPES(DECLARE_XIR_ATTRS_GET_MAP_SIZE) -XIR_ATTRS_SUPPORTED_PRIMITIVE_TYPES(DECLARE_XIR_ATTRS_GET_MAP_KEYS) -XIR_ATTRS_SUPPORTED_PRIMITIVE_TYPES(DECLARE_XIR_ATTRS_GET_MAP) -XIR_ATTRS_SUPPORTED_PRIMITIVE_TYPES(DECLARE_XIR_ATTRS_SET_MAP) -XIR_ATTRS_SUPPORTED_PRIMITIVE_TYPES(DECLARE_XIR_ATTRS_HAS_MAP) - -#define DECLARE_XIR_ATTRS_GET_MAP_VEC_MSIZE(type_name, c_type, cxx_type) \ -size_t xir_attrs_get_map_vec_msize_ ##type_name (xir_attrs_t attrs, const char* name); -#define DECLARE_XIR_ATTRS_GET_MAP_VEC_VSIZE(type_name, c_type, cxx_type) \ -size_t xir_attrs_get_map_vec_vsize_ ##type_name (xir_attrs_t attrs, const char* name, const char * key); -#define DECLARE_XIR_ATTRS_GET_MAP_VEC_KEYS(type_name, c_type, cxx_type) \ -void xir_attrs_get_map_vec_keys_ ##type_name (xir_attrs_t attrs, const char* name, const char *keys[]); -#define DECLARE_XIR_ATTRS_GET_MAP_VEC(type_name, c_type, cxx_type) \ -c_type xir_attrs_get_map_vec_ ##type_name (xir_attrs_t attrs, const char* name, const char * key, size_t idx); -#define DECLARE_XIR_ATTRS_SET_MAP_VEC(type_name, c_type, cxx_type) \ -void xir_attrs_set_map_vec_ ##type_name (xir_attrs_t attrs, const char* name, const char * key, size_t idx, c_type value); -#define DECLARE_XIR_ATTRS_HAS_MAP_VEC(type_name, c_type, cxx_type) \ -int xir_attrs_has_map_vec_ ##type_name (xir_attrs_t attrs, const char* name); - -XIR_ATTRS_SUPPORTED_PRIMITIVE_TYPES(DECLARE_XIR_ATTRS_GET_MAP_VEC_MSIZE) -XIR_ATTRS_SUPPORTED_PRIMITIVE_TYPES(DECLARE_XIR_ATTRS_GET_MAP_VEC_VSIZE) -XIR_ATTRS_SUPPORTED_PRIMITIVE_TYPES(DECLARE_XIR_ATTRS_GET_MAP_VEC_KEYS) -XIR_ATTRS_SUPPORTED_PRIMITIVE_TYPES(DECLARE_XIR_ATTRS_GET_MAP_VEC) -XIR_ATTRS_SUPPORTED_PRIMITIVE_TYPES(DECLARE_XIR_ATTRS_SET_MAP_VEC) -XIR_ATTRS_SUPPORTED_PRIMITIVE_TYPES(DECLARE_XIR_ATTRS_HAS_MAP_VEC) - -// clang-format on -/** - * @brief Check the existence of the attribute indicated by key. - * - * @param key A string to indicate the key of the attribute to check. - * - * @return 1 for existing, otherwise for not. - */ -int xir_attrs_has_attr(xir_attrs_t attrs, const char* key); -/** - * @brief Create a Tensor instance. - * - * @param name The name of the tensor. - * - * @param dims A array to indicate the tensor's dimensions. - * - * @param dim_num number of dimensions. - * - * @param data_type Indicates the type of the Tensor data. - * - * @param bit_width Indicates the bit width of the Tensor data. - * - * @return A unique pointer to the new Tensor object. - */ -xir_tensor_t xir_tensor_create(const char* name, const int32_t* dims, - const int32_t dim_num, - enum xir_tensor_data_type_t data_type, - const int32_t bit_width); - -/** - * @brief destroy a tensor - * - * @return return value is not in use yet. it is always zero. - */ -int xir_tensor_destroy(xir_tensor_t tensor); - -/** - * @brief Get name of the tensor. - * @param tensor - * @return The name of tensor . - */ -const char* xir_tensor_get_name(xir_tensor_t tensor); -/** - * @brief Get the bit width of the data in tensor. - * - * @return The bit width of the data in tensor. - */ -int32_t xir_tensor_get_bit_width(xir_tensor_t tensor); - -/** - * @brief Get the dimension size of one specific dimension indicated by idx. - * - * @param idx Indicate the dimension requested. - * - * @return The dimension size. - */ -int32_t xir_tensor_get_dim_size(xir_tensor_t tensor, int32_t idx); - -/** - * @brief Get the number of dimensions of the current Tensor object. - * - * @return The number of dimensions. - */ -int32_t xir_tensor_get_dim_num(xir_tensor_t tensor); - -/** - * @brief Get the data type of the tensor. - * - * @return Data type. - */ -enum xir_tensor_data_type_t xir_tensor_get_data_type(xir_tensor_t tensor); -/** - * @brief Get the number of elements in the tensor. - * - * @return Number of elements. - */ -int32_t xir_tensor_get_element_num(xir_tensor_t tensor); -/** - * @brief Get the number of data in the tensor. - * - * @return Number of data. - */ -int32_t xir_tensor_get_data_size(xir_tensor_t tensor); -/** - * @brief Get the Attrs object of the tensor. - * - * @return A unique pointer to the attrs - */ -xir_attrs_t xir_tensor_get_attrs(xir_tensor_t tensor); - -/** - * @brief Get num of the keys in this Attrs object. - * - * @return num of keys - */ -size_t xir_attrs_get_num_of_keys(xir_attrs_t attrs); - -/** - * @brief Get a key - * - * @return the key - */ -const char* xir_attrs_get_key(xir_attrs_t attrs, size_t idx); -/** - * @brief Get the name of subgraph. - * - * @return The name of the subgraph. - */ -const char* xir_subgraph_get_name(xir_subgraph_t subgraph); -/** - * @brief Get the number of children subgraphs. - * - * @return The number of the children subgraphs. - */ -int32_t xir_subgraph_get_children_num(xir_subgraph_t subgraph); -/** - * @brief Get the child subgraph of the current subgraph by idx. - * - * @return A raw pointer to a child subgraph. - */ -xir_subgraph_t xir_subgraph_get_child(xir_subgraph_t subgraph, int32_t idx); - -/** - * @brief Get all the children subgraphs of the current subgraph in the - * topological order. - * @param subgraph The current subgraph - * @param children All thr children subgraphs of the current subgraph int the - * topological order. - * @return A vector of the raw pointer of children subgraphs. - */ -void xir_subgraph_children_topological_sort(xir_subgraph_t subgraph, - xir_subgraph_t children[]); -#ifdef __cplusplus -} -#endif - -/* Local Variables: */ -/* mode:c */ -/* c-basic-offset: 2 */ -/* coding: undecided-unix */ -/* End: */ diff -pruN 1.4-2/src/python/CMakeLists.txt 2.5-1/src/python/CMakeLists.txt --- 1.4-2/src/python/CMakeLists.txt 2021-07-27 05:41:52.000000000 +0000 +++ 2.5-1/src/python/CMakeLists.txt 2022-06-15 07:33:00.000000000 +0000 @@ -14,5 +14,5 @@ vai_add_pybind11_module(wrapper MODULE_NAME xir wrapper/wrapper.cpp wrapper/pyxir_error_code.cpp) -target_link_libraries(wrapper PRIVATE xir) -target_include_directories(wrapper PRIVATE ${CMAKE_SOURCE_DIR}/src) +target_link_libraries(wrapper PRIVATE xir unilog::unilog) +target_include_directories(wrapper PRIVATE ${PROJECT_SOURCE_DIR}/src) diff -pruN 1.4-2/src/python/wrapper/wrapper.cpp 2.5-1/src/python/wrapper/wrapper.cpp --- 1.4-2/src/python/wrapper/wrapper.cpp 2021-07-27 05:41:52.000000000 +0000 +++ 2.5-1/src/python/wrapper/wrapper.cpp 2022-06-15 07:33:00.000000000 +0000 @@ -352,6 +352,11 @@ xir::Op* add_op(xir::Graph* graph, const if (!py::isinstance(py_attrs)) { UNI_LOG_CHECK(py::isinstance(py_attrs), PYXIR_INVALID_DATA_TYPE) << "attrs should be a dict"; + auto build_in_ops = xir::op_def_factory()->get_registered_ops(); + if (std::find(build_in_ops.begin(), build_in_ops.end(), type) == + build_in_ops.end()) { + xir::register_customized_operator_definition(name, type); + } auto defs = xir::op_def_factory()->create(type)->attrs(); for (auto attr : py::cast(py_attrs)) { auto key = py::cast(attr.first); diff -pruN 1.4-2/src/xir/attrs/attr_expander_imp.cpp 2.5-1/src/xir/attrs/attr_expander_imp.cpp --- 1.4-2/src/xir/attrs/attr_expander_imp.cpp 2021-07-27 05:41:52.000000000 +0000 +++ 2.5-1/src/xir/attrs/attr_expander_imp.cpp 2022-06-15 07:33:00.000000000 +0000 @@ -17,11 +17,11 @@ #include "xir/attrs/attr_expander_imp.hpp" #include "UniLog/UniLog.hpp" -#include #include #include #include #include +#include "xir/util/dynamic_load.hpp" namespace xir { diff -pruN 1.4-2/src/xir/attrs/attrs.cpp 2.5-1/src/xir/attrs/attrs.cpp --- 1.4-2/src/xir/attrs/attrs.cpp 2021-07-27 05:41:52.000000000 +0000 +++ 2.5-1/src/xir/attrs/attrs.cpp 2022-06-15 07:33:00.000000000 +0000 @@ -99,277 +99,3 @@ int Attrs::cmp(const any& a, const any& return it_cmp->second(a, b); } } // namespace xir - -// C API implementations -#include - -#include "xir/xir.h" -extern "C" xir_attrs_t xir_attrs_create() { - return static_cast(xir::Attrs::create().release()); -} - -extern "C" void xir_attrs_destroy(xir_attrs_t attrs) { - delete static_cast(attrs); -} - -// p(type_name, c_type, c++ type ) -template -struct convert { - static to_t conv(const from_t& value) { return value; } -}; - -template <> -struct convert { - static std::string conv(xir_string_t value) { return std::string(value); } -}; -template <> -struct convert { - static xir_string_t conv(const std::string& value) { return value.c_str(); } -}; -template <> -struct convert> { - static std::vector conv(const xir_bytes_t& value) { - auto ret = std::vector(value.size); - std::memcpy(&ret[0], value.data, value.size); - return ret; - } -}; -template <> -struct convert, xir_bytes_t> { - static xir_bytes_t conv(const std::vector& value) { - return xir_bytes_t{(int8_t*)(&value[0]), value.size()}; - } -}; -template <> -struct convert { - static bool conv(const int& value) { return value != 0; } -}; - -#define IMPL_GET(type_name, c_type, cxx_type) \ - extern "C" c_type xir_attrs_get_##type_name(xir_attrs_t attrs, \ - const char* name) { \ - return convert::conv( \ - static_cast(attrs)->get_attr( \ - std::string(name))); \ - } -XIR_ATTRS_SUPPORTED_PRIMITIVE_TYPES(IMPL_GET); - -#define IMPL_SET(type_name, c_type, cxx_type) \ - extern "C" void xir_attrs_set_##type_name(xir_attrs_t attrs, \ - const char* name, c_type value) { \ - static_cast(attrs)->set_attr( \ - std::string(name), convert::conv(value)); \ - } -XIR_ATTRS_SUPPORTED_PRIMITIVE_TYPES(IMPL_SET); - -#define IMPL_HAS(type_name, c_type, cxx_type) \ - extern "C" int xir_attrs_has_##type_name(xir_attrs_t attrs, \ - const char* name) { \ - auto self = static_cast(attrs); \ - return self->has_attr(std::string(name)) && \ - self->get_attr(std::string(name)).type() == typeid(cxx_type); \ - } -XIR_ATTRS_SUPPORTED_PRIMITIVE_TYPES(IMPL_HAS); - -extern int xir_attrs_has_attr(xir_attrs_t attrs, const char* key) { - return static_cast(attrs)->has_attr(std::string(key)); -} - -#define IMPL_VEC_GET_SIZE(type_name, c_type, cxx_type) \ - extern "C" size_t xir_attrs_get_vec_size_##type_name(xir_attrs_t attrs, \ - const char* name) { \ - return static_cast(attrs) \ - ->get_attr&>(std::string(name)) \ - .size(); \ - } - -XIR_ATTRS_SUPPORTED_PRIMITIVE_TYPES(IMPL_VEC_GET_SIZE); - -#define IMPL_VEC_GET(type_name, c_type, cxx_type) \ - extern "C" c_type xir_attrs_get_vec_##type_name( \ - xir_attrs_t attrs, const char* name, size_t idx) { \ - return convert::conv( \ - static_cast(attrs) \ - ->get_attr&>(std::string(name))[idx]); \ - } - -XIR_ATTRS_SUPPORTED_PRIMITIVE_TYPES(IMPL_VEC_GET); -#define IMPL_VEC_SET(type_name, c_type, cxx_type) \ - extern "C" void xir_attrs_set_vec_##type_name( \ - xir_attrs_t attrs, const char* name, size_t idx, c_type value) { \ - auto self = static_cast(attrs); \ - if (!self->has_attr(std::string(name))) { \ - self->set_attr>(std::string(name), \ - std::vector(idx + 1u)); \ - } \ - std::vector& vec = const_cast&>( \ - self->get_attr&>(std::string(name))); \ - if (idx >= vec.size()) { \ - vec.resize(idx + 1u); \ - } \ - vec[idx] = convert::conv(value); \ - } - -XIR_ATTRS_SUPPORTED_PRIMITIVE_TYPES(IMPL_VEC_SET); - -#define IMPL_VEC_HAS(type_name, c_type, cxx_type) \ - extern "C" int xir_attrs_has_vec_##type_name(xir_attrs_t attrs, \ - const char* name) { \ - auto self = static_cast(attrs); \ - return self->has_attr(std::string(name)) && \ - self->get_attr(std::string(name)).type() == \ - typeid(std::vector); \ - } - -XIR_ATTRS_SUPPORTED_PRIMITIVE_TYPES(IMPL_VEC_HAS); - -#define IMP_XIR_ATTRS_GET_MAP_SIZE(type_name, c_type, cxx_type) \ - extern "C" size_t xir_attrs_get_map_size_##type_name(xir_attrs_t attrs, \ - const char* name) { \ - auto self = static_cast(attrs); \ - return self \ - ->get_attr&>(std::string(name)) \ - .size(); \ - } -XIR_ATTRS_SUPPORTED_PRIMITIVE_TYPES(IMP_XIR_ATTRS_GET_MAP_SIZE) - -#define IMP_XIR_ATTRS_GET_MAP_KEYS(type_name, c_type, cxx_type) \ - extern "C" void xir_attrs_get_map_keys_##type_name( \ - xir_attrs_t attrs, const char* name, const char* keys[]) { \ - auto self = static_cast(attrs); \ - auto& m = self->get_attr&>( \ - std::string(name)); \ - int c = 0; \ - for (auto& k : m) { \ - keys[c++] = k.first.c_str(); \ - } \ - } -XIR_ATTRS_SUPPORTED_PRIMITIVE_TYPES(IMP_XIR_ATTRS_GET_MAP_KEYS) - -#define IMPL_MAP_GET(type_name, c_type, cxx_type) \ - extern "C" c_type xir_attrs_get_map_##type_name( \ - xir_attrs_t attrs, const char* name, const char* key) { \ - auto& it = (static_cast(attrs) \ - ->get_attr&>( \ - std::string(name)) \ - .at(std::string(key))); \ - return convert::conv(it); \ - } - -XIR_ATTRS_SUPPORTED_PRIMITIVE_TYPES(IMPL_MAP_GET); -#define IMPL_MAP_SET(type_name, c_type, cxx_type) \ - extern "C" void xir_attrs_set_map_##type_name( \ - xir_attrs_t attrs, const char* name, const char* key, c_type value) { \ - auto self = static_cast(attrs); \ - if (!self->has_attr(std::string(name))) { \ - self->set_attr>( \ - std::string(name), std::map()); \ - } \ - std::map& m = \ - const_cast&>( \ - self->get_attr&>( \ - std::string(name))); \ - m[std::string(key)] = convert::conv(value); \ - } - -XIR_ATTRS_SUPPORTED_PRIMITIVE_TYPES(IMPL_MAP_SET); - -#define IMPL_MAP_HAS(type_name, c_type, cxx_type) \ - extern "C" int xir_attrs_has_map_##type_name(xir_attrs_t attrs, \ - const char* name) { \ - auto self = static_cast(attrs); \ - return self->has_attr(std::string(name)) && \ - self->get_attr(std::string(name)).type() == \ - typeid(std::map); \ - } - -XIR_ATTRS_SUPPORTED_PRIMITIVE_TYPES(IMPL_MAP_HAS); - -#define IMP_XIR_ATTRS_GET_MAP_VEC_MSIZE(type_name, c_type, cxx_type) \ - extern "C" size_t xir_attrs_get_map_vec_msize_##type_name( \ - xir_attrs_t attrs, const char* name) { \ - auto self = static_cast(attrs); \ - return self \ - ->get_attr>&>( \ - std::string(name)) \ - .size(); \ - } -XIR_ATTRS_SUPPORTED_PRIMITIVE_TYPES(IMP_XIR_ATTRS_GET_MAP_VEC_MSIZE) - -#define IMP_XIR_ATTRS_GET_MAP_VEC_VSIZE(type_name, c_type, cxx_type) \ - extern "C" size_t xir_attrs_get_map_vec_vsize_##type_name( \ - xir_attrs_t attrs, const char* name, const char* key) { \ - auto self = static_cast(attrs); \ - return self \ - ->get_attr>&>( \ - std::string(name)) \ - .at(std::string(key)) \ - .size(); \ - } -XIR_ATTRS_SUPPORTED_PRIMITIVE_TYPES(IMP_XIR_ATTRS_GET_MAP_VEC_VSIZE) - -#define IMP_XIR_ATTRS_GET_MAP_VEC_KEYS(type_name, c_type, cxx_type) \ - extern "C" void xir_attrs_get_map_vec_keys_##type_name( \ - xir_attrs_t attrs, const char* name, const char* keys[]) { \ - auto self = static_cast(attrs); \ - auto& m = \ - self->get_attr>&>( \ - std::string(name)); \ - int c = 0; \ - for (auto& k : m) { \ - keys[c++] = k.first.c_str(); \ - } \ - } -XIR_ATTRS_SUPPORTED_PRIMITIVE_TYPES(IMP_XIR_ATTRS_GET_MAP_VEC_KEYS) - -#define IMPL_MAP_VEC_GET(type_name, c_type, cxx_type) \ - extern "C" c_type xir_attrs_get_map_vec_##type_name( \ - xir_attrs_t attrs, const char* name, const char* key, size_t idx) { \ - auto& it = \ - (static_cast(attrs) \ - ->get_attr>&>( \ - std::string(name)) \ - .at(std::string(key))); \ - return convert::conv(it[idx]); \ - } - -XIR_ATTRS_SUPPORTED_PRIMITIVE_TYPES(IMPL_MAP_VEC_GET); -#define IMPL_MAP_VEC_SET(type_name, c_type, cxx_type) \ - extern "C" void xir_attrs_set_map_vec_##type_name( \ - xir_attrs_t attrs, const char* name, const char* key, size_t idx, \ - c_type value) { \ - auto self = static_cast(attrs); \ - if (!self->has_attr(std::string(name))) { \ - self->set_attr>>( \ - std::string(name), std::map>()); \ - } \ - std::map>& m = const_cast< \ - std::map>&>( \ - self->get_attr>&>( \ - std::string(name))); \ - const auto& key_in_cxx = std::string(key); \ - auto it = m.find(key_in_cxx); \ - if (it == m.end()) { \ - bool ok = false; \ - std::tie(it, ok) = \ - m.insert({key_in_cxx, std::vector(idx + 1u)}); \ - } \ - auto& v = it->second; \ - if (idx >= v.size()) { \ - v.resize(idx + 1u); \ - } \ - v[idx] = convert::conv(value); \ - } - -XIR_ATTRS_SUPPORTED_PRIMITIVE_TYPES(IMPL_MAP_VEC_SET); - -#define IMPL_MAP_VEC_HAS(type_name, c_type, cxx_type) \ - extern "C" int xir_attrs_has_map_vec_##type_name(xir_attrs_t attrs, \ - const char* name) { \ - auto self = static_cast(attrs); \ - return self->has_attr(std::string(name)) && \ - self->get_attr(std::string(name)).type() == \ - typeid(std::map>); \ - } - -XIR_ATTRS_SUPPORTED_PRIMITIVE_TYPES(IMPL_MAP_VEC_HAS); diff -pruN 1.4-2/src/xir/attrs/attrs_imp.cpp 2.5-1/src/xir/attrs/attrs_imp.cpp --- 1.4-2/src/xir/attrs/attrs_imp.cpp 2021-07-27 05:41:52.000000000 +0000 +++ 2.5-1/src/xir/attrs/attrs_imp.cpp 2022-06-15 07:33:00.000000000 +0000 @@ -71,52 +71,4 @@ std::string AttrsImp::debug_info() const } return ret; } -/* -const std::vector AttrsImp::get_pbattr_keys() const { - std::vector ret; - - for (auto iter = attrs_.begin(); iter != attrs_.end(); ++iter) { - auto key = iter->first; - - if (key.size() >= 2 && key[0] == '_' && key[1] == '_') { - continue; - } - ret.push_back(key); - } - - return ret; - }*/ -class c_api { - public: - static size_t xir_attrs_get_num_of_keys(AttrsImp* self) { - return self->attrs_.size(); - } - static const char* xir_attrs_get_key(AttrsImp* self, size_t idx) { - auto num_of_buckets = self->attrs_.bucket_count(); - auto tmp_idx = 0u; - auto bucket_index = 0u; - for (bucket_index = 0u; - bucket_index < num_of_buckets && - idx >= tmp_idx + self->attrs_.bucket_size(bucket_index); - ++bucket_index) { - tmp_idx = tmp_idx + self->attrs_.bucket_size(bucket_index); - } - CHECK_LE(tmp_idx, idx); - auto it = self->attrs_.begin(bucket_index); - auto end = self->attrs_.end(bucket_index); - for (; tmp_idx < idx && it != end; tmp_idx++) { - ++it; - }; - return it->first.c_str(); - } -}; - } // namespace xir -#include "xir/xir.h" -extern "C" size_t xir_attrs_get_num_of_keys(xir_attrs_t attrs) { - return xir::c_api::xir_attrs_get_num_of_keys( - static_cast(attrs)); -} -extern "C" const char* xir_attrs_get_key(xir_attrs_t attrs, size_t idx) { - return xir::c_api::xir_attrs_get_key(static_cast(attrs), idx); -} diff -pruN 1.4-2/src/xir/attrs/attrs_imp.hpp 2.5-1/src/xir/attrs/attrs_imp.hpp --- 1.4-2/src/xir/attrs/attrs_imp.hpp 2021-07-27 05:41:52.000000000 +0000 +++ 2.5-1/src/xir/attrs/attrs_imp.hpp 2022-06-15 07:33:00.000000000 +0000 @@ -50,7 +50,6 @@ class AttrsImp : public Attrs { private: AttrMap attrs_; - friend class c_api; }; } // namespace xir diff -pruN 1.4-2/src/xir/CMakeLists.txt 2.5-1/src/xir/CMakeLists.txt --- 1.4-2/src/xir/CMakeLists.txt 2021-07-27 05:41:52.000000000 +0000 +++ 2.5-1/src/xir/CMakeLists.txt 2022-06-15 07:33:00.000000000 +0000 @@ -1,17 +1,17 @@ # # Copyright 2019 Xilinx Inc. # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. # include("${CMAKE_CURRENT_SOURCE_DIR}/../../cmake/VitisVersion.cmake") @@ -19,68 +19,92 @@ include("${CMAKE_CURRENT_SOURCE_DIR}/../ # get the git repo branch and commit id and write them into config.hpp execute_process( COMMAND git rev-parse --abbrev-ref HEAD - WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} OUTPUT_VARIABLE PROJECT_GIT_BRANCH_NAME OUTPUT_STRIP_TRAILING_WHITESPACE) execute_process( COMMAND git rev-parse HEAD - WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} OUTPUT_VARIABLE PROJECT_GIT_COMMIT_ID OUTPUT_STRIP_TRAILING_WHITESPACE) -if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 6.0) - set(CMAKE_CXX_COMPILER_VERSION_LESS_THAN_6_0 true CACHE BOOL - "Use GNU C++ compiler and version is less than 6.0") +if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION + VERSION_LESS 6.0) + set(CMAKE_CXX_COMPILER_VERSION_LESS_THAN_6_0 + true + CACHE BOOL "Use GNU C++ compiler and version is less than 6.0") endif() configure_file(config/config.hpp.in config.hpp @ONLY) # run protoc -set(protobuf_MODULE_COMPATIBLE ON CACHE BOOL "") +set(protobuf_MODULE_COMPATIBLE + ON + CACHE BOOL "") file(GLOB_RECURSE PROTO_FILES proto/*.proto) -protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS ${PROTO_FILES}) -set_source_files_properties(${PROTO_SRCS} PROPERTIES COMPILE_FLAGS -Wno-unused-variable) - +protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS EXPORT_MACRO XIR_DLLESPEC + ${PROTO_FILES}) +if(MSVC) + set_source_files_properties( + ${PROTO_SRCS} PROPERTIES COMPILE_DEFINITIONS + "XIR_DLLESPEC=__declspec(dllexport)") +else(MSVC) + set_source_files_properties( + ${PROTO_SRCS} + PROPERTIES COMPILE_DEFINITIONS + "XIR_DLLESPEC=__attribute__((visibility(\"default\")))") + set_source_files_properties(${PROTO_SRCS} PROPERTIES COMPILE_FLAGS + -Wno-unused-variable) +endif(MSVC) aux_source_directory(util/ CPP_SRCS) aux_source_directory(attrs/ CPP_SRCS) aux_source_directory(op/ CPP_SRCS) aux_source_directory(graph/ CPP_SRCS) aux_source_directory(tensor/ CPP_SRCS) +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/../../3rd-party/hash-library + CPP_SRCS) -add_library(${PROJECT_NAME} SHARED ${PROTO_SRCS} ${CPP_SRCS} ${CMAKE_CURRENT_BINARY_DIR}/version.c) -set_target_properties(${PROJECT_NAME} PROPERTIES - VERSION "${PROJECT_VERSION}" - SOVERSION "${PROJECT_VERSION_MAJOR}") -target_compile_features (${PROJECT_NAME} PUBLIC cxx_std_17) -target_include_directories(${PROJECT_NAME} - PUBLIC - $ - $ - PRIVATE - ${CMAKE_SOURCE_DIR}/src - ${CMAKE_CURRENT_BINARY_DIR}) -target_link_libraries(xir PRIVATE protobuf::libprotobuf pthread dl OpenSSL::Crypto PUBLIC unilog::unilog) +add_library(${PROJECT_NAME} SHARED ${PROTO_SRCS} ${CPP_SRCS} + ${CMAKE_CURRENT_BINARY_DIR}/version.c) +set_target_properties( + ${PROJECT_NAME} PROPERTIES VERSION "${PROJECT_VERSION}" + SOVERSION "${PROJECT_VERSION_MAJOR}") +target_compile_features(${PROJECT_NAME} PUBLIC cxx_std_17) +target_include_directories( + ${PROJECT_NAME} + PUBLIC $ + $ + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/util ${Boost_INCLUDE_DIRS} + ${PROJECT_SOURCE_DIR}/src ${CMAKE_CURRENT_BINARY_DIR}) +target_link_libraries( + xir + PRIVATE protobuf::libprotobuf Threads::Threads ${CMAKE_DL_LIBS} + PUBLIC unilog::unilog) +# Enable Export of public APIs +target_compile_definitions(${PROJECT_NAME} PRIVATE -DXIR_EXPORT) install( TARGETS ${PROJECT_NAME} EXPORT ${PROJECT_NAME}-targets RUNTIME DESTINATION bin + ARCHIVE DESTINATION lib LIBRARY DESTINATION lib) -foreach(PUBLIC_HEADER - xir/xir.h - xir/op/op.hpp - xir/op/op_def.hpp - xir/attrs/attrs.hpp - xir/attrs/attr_def.hpp - xir/attrs/attr_expander.hpp - xir/util/any.hpp - xir/util/tool_function.hpp - xir/util/data_type.hpp - xir/graph/graph.hpp - xir/graph/graph_template.hpp - xir/graph/subgraph.hpp - xir/tensor/tensor.hpp) +foreach( + PUBLIC_HEADER + xir/XirExport.hpp + xir/op/op.hpp + xir/op/op_def.hpp + xir/attrs/attrs.hpp + xir/attrs/attr_def.hpp + xir/attrs/attr_expander.hpp + xir/util/any.hpp + xir/util/tool_function.hpp + xir/util/data_type.hpp + xir/graph/graph.hpp + xir/graph/graph_template.hpp + xir/graph/subgraph.hpp + xir/tensor/tensor.hpp) get_filename_component(HEADER_PATH ${PUBLIC_HEADER} DIRECTORY) - install(FILES ${CMAKE_SOURCE_DIR}/include/${PUBLIC_HEADER} - DESTINATION include/${HEADER_PATH}) + install(FILES ${PROJECT_SOURCE_DIR}/include/${PUBLIC_HEADER} + DESTINATION include/${HEADER_PATH}) endforeach() install( EXPORT ${PROJECT_NAME}-targets diff -pruN 1.4-2/src/xir/graph/elf2xir.cpp 2.5-1/src/xir/graph/elf2xir.cpp --- 1.4-2/src/xir/graph/elf2xir.cpp 2021-07-27 05:41:52.000000000 +0000 +++ 2.5-1/src/xir/graph/elf2xir.cpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,1146 +0,0 @@ -/* - * Copyright 2019 Xilinx Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#ifdef __QNX__ -#include -#else -#include -#endif - -#include -#include -#include -#include -#include -#include -template -struct env_config_helper { - static inline T from_string(const char* s); -}; - -template -struct env_config { - static T init() { - const char* name = env_name::get_name(); - const char* defvalue = env_name::get_default_value(); - const char* p = getenv(name); - const char* pstr = p != nullptr ? p : defvalue; - const T value = env_config_helper::from_string(pstr); - return value; - } - static T value; -}; -template -T env_config::value = env_config::init(); - -template <> -inline int env_config_helper::from_string(const char* s) { - return std::stoi(std::string(s)); -} -template <> -inline std::string env_config_helper::from_string(const char* s) { - return std::string(s); -} - -#define DEF_ENV_PARAM_2(param_name, defvalue1, type) \ - struct ENV_PARAM_##param_name \ - : public env_config { \ - static const char* get_name() { return #param_name; } \ - static const char* get_default_value() { return defvalue1; } \ - }; - -#define ENV_PARAM(param_name) (ENV_PARAM_##param_name::value) - -#define DEF_ENV_PARAM(param_name, defvalue1) \ - DEF_ENV_PARAM_2(param_name, defvalue1, int) - -DEF_ENV_PARAM(DEBUG_ELF_XIR, "0") -DEF_ENV_PARAM(DEBUG_ELF, "0") -// this struct is defined by elf format, try to be compatible -namespace { -struct tensor { - uint32_t attr; - uint32_t height; - uint32_t width; - uint32_t channel; - uint32_t addr_logical; - uint32_t size; - uint32_t fix_width; - int32_t fix_pos; - uint32_t channel_stride; - uint32_t reserved0; - uint32_t reserved1; - uint32_t reserved2; - uint32_t reserved3; -}; - -class buffer_object_fd { - public: - static std::shared_ptr create(const std::string& name, - int flags); - - public: - explicit buffer_object_fd(const std::string& name, int flags); - buffer_object_fd(const buffer_object_fd&) = delete; - buffer_object_fd& operator=(const buffer_object_fd& other) = delete; - virtual ~buffer_object_fd(); - - int fd() { return fd_; } - - private: - int fd_; -}; - -#define ELF_PHDR(name) \ - (is_class32() ? get(0)->name : get(0)->name) - -#define ELF_SHDR(i, name) \ - (is_class32() ? get(shoff())[i].name \ - : get(shoff())[i].name) - -class Elf { - public: - static std::unique_ptr create(const std::string& filename, - const std::string& kernel_name); - - static std::vector>> create( - const std::string& filename); - - public: - Elf(const std::string& filename, const std::string& kernel_name); - ~Elf(); - - public: - std::string Show(); - size_t CodeSize(); - int8_t* Code(); - size_t InitialCodeSize(); - int8_t* InitialCode(); - size_t ParameterSize(); - int8_t* Parameter(); - int8_t* Symtab(); - struct node; - const std::vector& Nodes() const { return nodes_; } - node* FindNodeByName(const char* name); - tensor* FindInputTensorByName(const char* name, int idx); - tensor* FindOutputTensorByName(const char* name, int idx); - const std::vector& kernels() const { return kernels_; } - - public: - struct metadata { - uint32_t dpu_arch; // - uint32_t ver_dnnc; // = 1; - uint32_t mode; // = 2; - uint32_t node_cnt; // = 3; - uint32_t tensor_size; // = 4; - uint32_t kernel_io_size; // = 5; - uint32_t kernel_mean_c1; // = 6; - uint32_t kernel_mean_c2; // = 7; - uint32_t kernel_mean_c3; // = 8; - uint16_t abi_ver_minor; // = // LSB(0~15): minor verion - uint16_t abi_ver_major; // = // MSB(16~31): major version - uint16_t dpu_ver_target; // = 10; // LSB(0~15): dpu target - uint16_t dpu_ver_arch_type; // = 10; // MSB(16~31): dpu arch type - uint32_t tensor_cnt; // = 11; - }; - template - struct list { - uint32_t cnt; - T data[]; - template - NextType* next() { - return reinterpret_cast(reinterpret_cast(this) + - cnt * sizeof(T) + sizeof(cnt)); - } - }; - struct node { - uint32_t type; - uint32_t name; - uint64_t workload; - struct reg { - uint16_t reg_id; - uint16_t data_type; - }; - uint32_t* marker[]; - list* regs() { return reinterpret_cast*>(&marker[0]); }; - list* inputs() { return regs()->next>(); } - list* outputs() { return inputs()->next>(); }; - struct code { - uint32_t offset; - uint32_t size; - uint32_t name_idx; - uint32_t align; - }; - list* codes() { return outputs()->next>(); }; - struct param { - uint32_t offset; - uint32_t size; - uint32_t fix_w; - uint32_t fix_p; - uint32_t name_idx; - uint32_t align; - uint32_t height; - uint32_t width; - uint32_t channel; - uint32_t out_channel; - }; - list* param() { // - return codes()->next>(); - } - list* pre_nodes() { return param()->next>(); } - list* suc_nodes() { return pre_nodes()->next>(); }; - }; - // for internal use, but still public - template - T* get(size_t offset) { - return reinterpret_cast(static_cast(data_) + offset); - } - bool is_class32() { - return get(0)->e_ident[EI_CLASS] == ELFCLASS32; - } - bool is_class64() { - return get(0)->e_ident[EI_CLASS] == ELFCLASS64; - } - - size_t shstrndx() { return ELF_PHDR(e_shstrndx); } - size_t shoff() { return ELF_PHDR(e_shoff); } - - size_t shnum() { return ELF_PHDR(e_shnum); } - - std::string deephi_string(size_t offset) { - return std::string( - get(section_offset(section_deephi_strtab_) + offset)); - } - - std::string strtab_string(size_t offset) { - return std::string(get(section_offset(section_strtab_) + offset)); - } - - std::string section_name(int section_idx) { - return std::string(get(ELF_SHDR(shstrndx(), sh_offset) + - ELF_SHDR(section_idx, sh_name))); - } - - size_t section_offset(int section_idx) { - return ELF_SHDR(section_idx, sh_offset); - } - - size_t section_size(int section_idx) { - return ELF_SHDR(section_idx, sh_size); - } - - size_t section_link(int section_idx) { - return ELF_SHDR(section_idx, sh_link); - } - - size_t section_entry_size(int section_idx) { - return ELF_SHDR(section_idx, sh_entsize); - } - - metadata* get_metadata() { - return get(section_offset(section_metadata_)); - } - - node* get_node() { // - return get(section_offset(section_node_)); - } - std::string get_file_name() { return filename_; } - std::string get_kernel_name() { return kernel_; } - std::string dump_meta_data(); - std::string dump_tensor(tensor* t); - void parse_sections(); - void parse_section(size_t i); - std::unique_ptr> get_preload_offset_and_size( - const std::string& layer); - size_t get_num_of_tensors() { // - return section_size(section_tensor_) / sizeof(tensor); - } - tensor* get_tensor(int i) { - return get(section_offset(section_tensor_) + i * sizeof(tensor)); - } - void parse_nodes(); - void parse_symtab(); - std::string dump_nodes(); - std::string dump_node(node* n); - - private: - std::string filename_; - std::shared_ptr fd_; - size_t size_; - void* data_; - std::string kernel_; - uint32_t section_metadata_; - uint32_t section_strtab_; - uint32_t section_deephi_strtab_; - uint32_t section_node_; - uint32_t section_parameter_; - uint32_t section_code_; - uint32_t section_initial_code_; - uint32_t section_tensor_; - uint32_t section_symtab_; - std::vector nodes_; - std::vector kernels_; -}; - -class ElfSectionBuilder; -struct ElfBuilder { - ElfBuilder(std::ostream& out) : out_{out} {}; - void build(); - ElfSectionBuilder* new_section(const std::string& name); - - private: - ElfSectionBuilder* get_section(const std::string& name); - ElfSectionBuilder* get_or_new_section(const std::string& name); - void prepare_build(); - void build_header(); - void build_section_headers(); - void build_section_header(ElfSectionBuilder* s); - void build_sections(); - void build_section(ElfSectionBuilder* s); - uint32_t total_section_size(); - - private: - std::ostream& out_; - std::vector> sections_; -}; - -struct ElfSectionBuilder { - public: - ElfSectionBuilder(const std::string& name, uint32_t id); - uint32_t allocate_string(const std::string& str); - void allocate_section_name(ElfSectionBuilder* str); - uint32_t allocate_section_space(uint32_t pos); - const std::string& get_name() const { return name_; } - uint32_t get_id() const { return id_; } - std::string data() const { return out_.str(); } - size_t size() { return out_.tellp(); } - uint32_t get_section_name(); - uint32_t offset(); - void align(); - void set_type(uint32_t type) { type_ = type; } - uint32_t get_type() const { return type_; } - template - void write(const X& x) { - CHECK(out_.write(reinterpret_cast(&x), sizeof(x)).good()); - } - void write(const std::vector& x) { - CHECK(out_.write(&x[0], x.size()).good()); - } - void write(const std::string& x) { - CHECK(out_.write(&x[0], x.size()).good()); - } - - private: - std::string name_; - uint32_t id_; - std::ostringstream out_; - uint32_t section_name_; - uint32_t offset_; - uint32_t type_; -}; - -struct DnncKernel { - public: - DnncKernel(const std::string& name, ElfBuilder* elf); - - public: - void set_kernel_iosize(size_t size) { kernel_io_size_ = size; } - void set_num_of_nodes(size_t size) { num_of_nodes_ = size; } - void build_meta_section(); - void build_code_section(const std::vector& code); - void build_parameter_section(const std::vector& parameter); - void build_node_section(); - void build_tensor_section(); - void add_tensor(const tensor& tensor) { tensors_.emplace_back(tensor); }; - void add_node(const std::function& builder); - uint32_t deephi_allocate_string(const std::string& str) { - return strtab_->allocate_string(str); - } - uint32_t find_tensor_by_ddr_addr(uint32_t ddr_addr); - - private: - ElfSectionBuilder* strtab_; - ElfSectionBuilder* metadata_; - ElfSectionBuilder* configurable_; - ElfSectionBuilder* tensor_; - ElfSectionBuilder* node_; - ElfSectionBuilder* parameter_; - ElfSectionBuilder* code_; - std::string name_; - size_t kernel_io_size_ = 0u; - std::vector tensors_ = {}; - size_t num_of_nodes_ = 0u; -}; -template -static inline std::ostream& operator<<(std::ostream& out, - const std::vector& v) { - int c = 0; - out << "["; - for (const auto x : v) { - if (c++ != 0) { - out << ","; - } - out << x; - } - out << "]"; - return out; -} - -using namespace std; - -template -struct WeakStore { - template - static std::shared_ptr create(const K& key, Args&&... args) { - static std::unordered_map> the_store_; - std::shared_ptr ret; - if (the_store_[key].expired()) { - ret = std::make_shared(std::forward(args)...); - the_store_[key] = ret; - } - ret = the_store_[key].lock(); - assert(ret != nullptr); - return ret; - } -}; - -static size_t get_size(int fd) { - struct stat statbuf; - const auto r_stat = fstat(fd, &statbuf); - CHECK_EQ(r_stat, 0) << "fstat error: "; - CHECK_GT(statbuf.st_size, 0) << "must not empty file"; - return statbuf.st_size; -} -static void* my_map(int fd, size_t size) { - auto p = mmap(NULL, size, PROT_READ, MAP_PRIVATE, fd, 0); - CHECK_NE(p, MAP_FAILED) << "cannot mmap"; - return p; -} - -std::shared_ptr buffer_object_fd::create( - const std::string& name, int flags) { - return WeakStore::create(name, name, flags); -} - -static int my_open(const std::string& name, int flags) { - auto fd = open(name.c_str(), flags); - CHECK_GT(fd, 0) << ", open(" << name << ") failed."; - return fd; -} - -buffer_object_fd::buffer_object_fd(const std::string& name, int flags) - : fd_{my_open(name, flags)} {} -buffer_object_fd::~buffer_object_fd() { close(fd_); } - -std::unique_ptr Elf::create(const std::string& filename, - const std::string& kernel_name) { - return std::unique_ptr(new Elf(filename, kernel_name)); -} - -std::vector>> Elf::create( - const std::string& filename) { - auto elf = Elf::create(filename, ""); - auto ret = std::vector>>{}; - for (const auto& k : elf->kernels()) { - ret.emplace_back(std::make_pair(k, Elf::create(filename, k))); - } - return ret; -} - -static std::shared_ptr create_fd( - const std::string& filename) { - auto ret = buffer_object_fd::create(filename, O_RDONLY | O_CLOEXEC); - CHECK_GT(ret->fd(), 0) << "cannot open filename"; - return ret; -}; -Elf::Elf(const std::string& filename, const std::string& kernel_name) - - : filename_{filename}, - fd_{create_fd(filename)}, - size_{get_size(fd_->fd())}, - data_{my_map(fd_->fd(), size_)}, - kernel_{kernel_name}, - section_metadata_{0}, - section_strtab_{0}, - section_deephi_strtab_{0}, - section_node_{0}, - section_parameter_{0}, - section_code_{0}, - section_initial_code_{0}, - section_tensor_{0}, - section_symtab_{0}, - nodes_() { - auto is_elf = get(0)->e_ident[EI_MAG0] == ELFMAG0 && - get(0)->e_ident[EI_MAG1] == ELFMAG1 && - get(0)->e_ident[EI_MAG2] == ELFMAG2 && - get(0)->e_ident[EI_MAG3] == ELFMAG3; - CHECK(is_elf == true) << ", " << filename << " is not a elf file"; - CHECK(is_class32() || is_class64()) - << ", only class32 or class64 is supported"; - parse_sections(); -} -Elf::~Elf() { - munmap(data_, size_); - LOG_IF(INFO, ENV_PARAM(DEBUG_ELF)) << "close elf file " << filename_; -} - -int8_t* Elf::Code() { return get(section_offset(section_code_)); } - -size_t Elf::ParameterSize() { return section_size(section_parameter_); } -int8_t* Elf::Parameter() { - return get(section_offset(section_parameter_)); -} - -void Elf::parse_sections() { - for (auto i = 1u; i < shnum(); ++i) { - parse_section(i); - } - parse_nodes(); - parse_symtab(); -} - -void Elf::parse_section(size_t idx) { - auto name = section_name(idx); - // LOG(INFO) << "parsing section[" << idx << "]" << name; - if (name == ".deephi.metadata." + kernel_) { - section_metadata_ = idx; - } else if (name == ".deephi.strtab." + kernel_) { - section_deephi_strtab_ = idx; - } else if (name == ".strtab") { - section_strtab_ = idx; - } else if (name == ".deephi.node." + kernel_) { - section_node_ = idx; - } else if (name == ".deephi.parameter." + kernel_) { - section_parameter_ = idx; - } else if (name == ".deephi.code." + kernel_) { - section_code_ = idx; - } else if (name == ".deephi.initial_code." + kernel_) { - section_initial_code_ = idx; - } else if (name == ".deephi.tensor." + kernel_) { - section_tensor_ = idx; - } else if (name == ".symtab") { - section_symtab_ = idx; - } else { - /* DLOG(INFO) << "unknown section " << section_name(idx) - << " filename = " << filename_; - */ - } - const char nn[] = ".deephi.metadata."; - const size_t start = sizeof(nn) - 1; - if (name.find(nn) == 0) { - kernels_.emplace_back(name.substr(start)); - } -} - -void Elf::parse_nodes() { - if (section_node_ == 0) { - // kernel name is not detected - return; - } - nodes_.resize(get_metadata()->node_cnt); - if (get_metadata()->node_cnt > 0) { - auto idx = 0u; - nodes_[0] = get(section_offset(section_node_)); - for (idx = 1; idx < nodes_.size(); ++idx) { - nodes_[idx] = nodes_[idx - 1]->suc_nodes()->next(); - } - } -} - -std::unique_ptr> Elf::get_preload_offset_and_size( - const std::string& layer) { - std::unique_ptr> ret = nullptr; - if (section_symtab_ == 0) { - return ret; - } - auto size = section_size(section_symtab_); - auto n_of_symbols = size / section_entry_size(section_symtab_); - LOG_IF(INFO, ENV_PARAM(DEBUG_ELF)) - << "section_symtab_ " << section_symtab_ << " " // - << " section_entry_size(section_symtab_) " - << "sizeof(Elf64_Sym) " << sizeof(Elf64_Sym) << " " // - << "sizeof(Elf32_Sym) " << sizeof(Elf32_Sym) << " " // - << section_entry_size(section_symtab_) << " " // - << "n_of_symbols " << n_of_symbols << " " // - ; - -#if defined __aarch64__ - auto symbols = get(section_offset(section_symtab_)); -#elif defined __arm__ - auto symbols = get(section_offset(section_symtab_)); -#elif defined __x86_64__ - auto symbols = get(section_offset(section_symtab_)); -#elif defined __microblaze__ - // I am not sure it is 64bits or 32 bits to avoid recompile all - // models, let's assume it is 64bits model - auto symbols = get(section_offset(section_symtab_)); -#else -#error "Platform not support!" -#endif - std::string name = - std::string("_dpu_") + kernel_ + "_" + layer + "_preload_code"; - for (auto i = 0u; i < n_of_symbols; ++i) { - if (strtab_string(symbols[i].st_name) == name) { - ret = std::unique_ptr>( - new std::pair(symbols[i].st_value, - symbols[i].st_size)); - break; - } - } - if (!ret && 0) { - LOG(WARNING) << "cannot find symbol " << name; - } - return ret; -} - -void Elf::parse_symtab() { - if (section_symtab_ == 0) { - return; - } - if (0) { - auto size = section_size(section_symtab_); - auto n_of_symbols = size / section_entry_size(section_symtab_); - LOG(INFO) << "section_symtab_ " << section_symtab_ << " " // - << " section_entry_size(section_symtab_) " - << "sizeof(Elf64_Sym) " << sizeof(Elf64_Sym) << " " // - << "sizeof(Elf32_Sym) " << sizeof(Elf32_Sym) << " " // - << section_entry_size(section_symtab_) << " " // - << "n_of_symbols " << n_of_symbols << " " // - ; - /* -#if defined __aarch64__ - auto symbols = get(section_offset(section_symtab_)); -#elif defined __arm__ - auto symbols = get(section_offset(section_symtab_)); -#elif defined __x86_64__ - auto symbols = get(section_offset(section_symtab_)); -#else -#error "Platform not support!" -#endif - */ -#if defined __arm__ - auto symbols = get(section_offset(section_symtab_)); -#else - auto symbols = get(section_offset(section_symtab_)); -#endif - for (auto i = 0u; i < n_of_symbols; ++i) { - LOG(INFO) << std::hex << "symbols[i].st_name " - << strtab_string(symbols[i].st_name) << " " // - << std::dec << "symbols[i].st_value " << symbols[i].st_value - << " " // - << "symbols[i].st_size " << symbols[i].st_size << " " // - << "symbols[i].st_shndx " << symbols[i].st_shndx << " " // - ; - } - auto x = get_preload_offset_and_size("inception_b1_1x7_reduce"); - if (x) { - LOG(INFO) << "x->first " << x->first << " " // - << "x->second " << x->second << " " // - ; - } - } -} - -struct elf2xir { - const char* DPU_OP_TYPE = "concat"; - const char* DPU_INPUT_OP = "concat"; - const std::string filename_; - const std::string kernel_; - std::vector input_ops_name_; - std::vector output_ops_name_; - Elf* elf_; - Elf::metadata* meta_data_; - xir::Op* previous_op_; - xir::Op* last_op_; - xir::Graph* graph_; - std::map tensor_id_2_node_id_; - std::map node_id_2_kernel_id_; - std::map> tensor_id_2_input_tensor_ids_; - std::map tensor_id_2_is_input_; - std::map tensor_id_2_tf_node_; - size_t io_size_ = 0; - std::vector parameter_; - elf2xir(Elf* elf) - : filename_(elf->get_file_name()), - kernel_(elf->get_kernel_name()), - input_ops_name_{}, - output_ops_name_{}, - elf_{std::move(elf)}, - meta_data_{elf_->get_metadata()} { - // LOG(INFO) << "kernel name is " << kernel_; - CHECK(!filename_.empty()) << "FILENAME = " << elf->get_file_name(); - } - - private: - void add_metadata() { - auto debug_mode = meta_data_->mode != 0; - CHECK(!debug_mode) << "debug mode not supported yet."; - auto io_size = meta_data_->kernel_io_size; - io_size_ = io_size; - auto parameter_size = elf_->ParameterSize(); - auto parameter = elf_->Parameter(); - parameter_.assign(parameter, parameter + parameter_size); - return; - } - - enum RtNodeType { DpuCodeNode = 1, DpuVirtNode = 2, CpuNode = 0x80 }; - enum RtTensorType { - NormalTensor = (1 << 0), - InputEdgeTensor = (1 << 1), - OutputEdgeTensor = (1 << 2), - }; - - string tf_node_name(int tensor_id) { - auto it_node_id = tensor_id_2_node_id_.find(tensor_id); - auto found_node_id = it_node_id != tensor_id_2_node_id_.end(); - auto node_id = found_node_id ? it_node_id->second : -1; - auto node_name = found_node_id - ? elf_->deephi_string(elf_->Nodes()[node_id]->name) - : kernel_ + string("_INPUT"); - return node_name; - } - - string node_name(int node_id) { - auto node_name = elf_->deephi_string(elf_->Nodes()[node_id]->name); - return node_name; - } - std::unique_ptr create_xir_tensor(const std::string& name, - const tensor* tensor) { - CHECK(tensor != nullptr) << "not a tensor! name=" << name; - auto batch = 1; - auto height = (int)tensor->height; - auto width = (int)tensor->width; - auto channel = (int)tensor->channel; - auto ret = xir::Tensor::create(name, {batch, height, width, channel}, - xir::DataType{xir::DataType::XINT, 8}); - auto reg_id = 1; // REG ID is always 1 for now; - auto ddr_addr = tensor->addr_logical; - auto location = 1; // always on ddr; - auto fix_pos = tensor->fix_pos; - ret->set_attr("reg_id", reg_id); - ret->set_attr("ddr_addr", (std::int32_t)ddr_addr); - ret->set_attr("location", location); - ret->set_attr("fix_point", fix_pos); - LOG_IF(INFO, ENV_PARAM(DEBUG_ELF_XIR)) - << "create tensor: " - << "name " << name << " " // - << "ddr_addr " << std::hex << "0x" << ddr_addr << std::dec << " " // - << "fixpoint is " << fix_pos; - return ret; - } - - void connect_tf_nodes() { - auto n_of_tensors = elf_->get_num_of_tensors(); - for (auto tensor_id = 0u; tensor_id < n_of_tensors; ++tensor_id) { - auto it_node_id = tensor_id_2_node_id_.find(tensor_id); - auto found_node_id = it_node_id != tensor_id_2_node_id_.end(); - if (found_node_id) { - auto node_id = it_node_id->second; - auto& input_tensor_ids = tensor_id_2_input_tensor_ids_[tensor_id]; - auto node = elf_->Nodes()[node_id]; - for (auto i = 0u; i < node->inputs()->cnt; ++i) { - auto input_tensor_id = node->inputs()->data[i]; - input_tensor_ids.emplace_back(input_tensor_id); - } - } - } - } - - uint32_t get_4k_align_size(uint32_t code_size) { - return (code_size & 0xfff) ? (((code_size >> 12) + 1) << 12) : code_size; - } - - std::string code_buffer = ""; - // uint32_t node_code_offset = 0; - // int preload_size = 0; - - std::map build_tensor_id_2_node_id() { - int node_id = 0; - const auto& nodes = elf_->Nodes(); - auto ret = std::map{}; - for (const auto& node : nodes) { - for (auto i = 0u; i < node->outputs()->cnt; ++i) { - auto tensor_id = node->outputs()->data[i]; - CHECK(ret.find(tensor_id) == ret.end()) - << " tensor id must belong to at most one node id. tensor_id = " - << tensor_id << " , node_id=" << node_id; - ret[tensor_id] = node_id; - } - node_id++; - } - return ret; - } - - std::map build_node_id_2_kernel_id() { - auto ret = std::map{}; - auto debug_mode = meta_data_->mode != 0; - const auto& nodes = elf_->Nodes(); - int node_id = 0; - int kernel_id = 1; - for (const auto& node : nodes) { - if (node->type == DpuCodeNode) { - if (debug_mode) { - ret[node_id] = kernel_id; - } else { - ret[node_id] = 0; // all dpu kernel has the same kernel id - } - } else if (node->type == DpuVirtNode) { - if (debug_mode) { - ret[node_id] = kernel_id; - } else { - ret[node_id] = 0; // all dpu kernel has the same kernel id - } - } else if (node->type == CpuNode) { - ret[node_id] = kernel_id; - } else { - LOG(FATAL) << "unknown node type, id=" << node_id; - } - kernel_id++; - node_id++; - } - return ret; - } - template - static std::string dump_object(const T& v) { - std::ostringstream out; - out << v; - return out.str(); - } - xir::Op* create_dpu_op(unsigned int tensor_id) { - auto it_input_tensor_ids = tensor_id_2_input_tensor_ids_.find(tensor_id); - CHECK(it_input_tensor_ids != tensor_id_2_input_tensor_ids_.end()) - << "cannot find input op! tensor_id=" << tensor_id; - auto& input_tensor_ids = it_input_tensor_ids->second; - auto input_ops = std::vector(); - for (auto idx = 0u; idx < input_tensor_ids.size(); ++idx) { - auto input_tensor_id = input_tensor_ids[idx]; - auto it_input_op = tensor_id_2_tf_node_.find(input_tensor_id); - CHECK(it_input_op != tensor_id_2_tf_node_.end()) - << "cannot find input op! tensor_id=" << tensor_id - << ", input_tensor_id=" << input_tensor_id; - auto op = it_input_op->second; - input_ops.emplace_back(op); - } - // - // dirty hack, all op are concat - auto attrs = xir::Attrs::create(); - attrs->set_attr("axis", 0); - auto tensor_op_name = - kernel_ + std::string("_op_") + std::to_string(tensor_id); - auto tensor = elf_->get_tensor(tensor_id); - last_op_ = graph_->add_op(tensor_op_name, std::string(DPU_OP_TYPE), - std::move(attrs), {{"input", input_ops}}); - last_op_->replace_output_tensor(create_xir_tensor( - tf_node_name(tensor_id) + "_" + std::to_string(tensor_id), tensor)); - return last_op_; - } - - xir::Op* create_input_op(unsigned int tensor_id) { - auto tensor_op_name = - kernel_ + std::string("_op_") + std::to_string(tensor_id); - auto tensor = elf_->get_tensor(tensor_id); - auto attrs = xir::Attrs::create(); - attrs->set_attr("axis", 0); - auto xir_tensor = create_xir_tensor( - tf_node_name(tensor_id) + "_" + std::to_string(tensor_id), tensor); - auto ret = graph_->add_op(tensor_op_name, std::string(DPU_INPUT_OP), - std::move(attrs), {{"input", {previous_op_}}}); - ret->replace_output_tensor(std::move(xir_tensor)); - return ret; - } - - void add_tf_nodes() { - auto n_of_tensors = elf_->get_num_of_tensors(); - LOG_IF(INFO, ENV_PARAM(DEBUG_ELF_XIR)) - << " filename = " << filename_ << " " - << " kernel_name=" << elf_->get_kernel_name() - << " n_of_tensors=" << n_of_tensors; - for (auto tensor_id = 0u; tensor_id < n_of_tensors; ++tensor_id) { - auto node_name = tf_node_name(tensor_id); - auto it_node_id = tensor_id_2_node_id_.find(tensor_id); - auto found_node_id = it_node_id != tensor_id_2_node_id_.end(); - auto node_id = found_node_id ? it_node_id->second : -1; - auto it_kernel_id = found_node_id ? node_id_2_kernel_id_.find(node_id) - : node_id_2_kernel_id_.end(); - auto found_kernel_id = it_kernel_id != node_id_2_kernel_id_.end(); - auto kernel_id = found_kernel_id ? it_kernel_id->second : -1; - auto tensor = elf_->get_tensor(tensor_id); - auto is_input = false; - auto is_output = false; - auto op_type = std::string(DPU_OP_TYPE); - if (tensor->attr == NormalTensor) { - is_input = false; - is_output = false; - op_type = std::string(DPU_OP_TYPE); - } else if (tensor->attr == InputEdgeTensor) { - is_input = true; - is_output = false; - op_type = std::string(DPU_INPUT_OP) + "_input"; // ugly code - } else if (tensor->attr == OutputEdgeTensor) { - is_input = false; - is_output = true; - op_type = std::string(DPU_OP_TYPE); - } else { - LOG(FATAL) << "Unkonwn tensor attr tensor_id =" << tensor_id; - } - - auto tf_node = op_type == std::string(DPU_OP_TYPE) - ? create_dpu_op(tensor_id) - : create_input_op(tensor_id); - tensor_id_2_tf_node_[tensor_id] = tf_node; - - if (found_node_id) { - auto elf_node = elf_->Nodes()[node_id]; - if (elf_node->type == DpuCodeNode) { - } else if (elf_node->type == DpuVirtNode) { - } else if (elf_node->type == CpuNode) { - LOG(FATAL) << "unsupported node type, id=" << node_id; - } else { - LOG(FATAL) << "unknown node type, id=" << node_id; - } - } else { - CHECK_EQ(tf_node->get_type(), DPU_INPUT_OP); - } - tf_node->set_attr("is_graph_input", is_input); - tensor_id_2_is_input_[tensor_id] = is_input; - tf_node->set_attr("is_graph_output", is_output); - tf_node->set_attr("fix_point", tensor->fix_pos); - tf_node->set_attr( - "super_layer_id", - kernel_ + std::string("_") + (found_node_id ? node_name : "Input")); - tf_node->set_attr("kernel_id", - kernel_ + std::string("_") + std::to_string(kernel_id)); - if (kernel_id == -1) { - tf_node->set_attr("node_type", "input"); - } else { - tf_node->set_attr("node_type", "node"); - } - tf_node->set_attr("node_id", node_id); - tf_node->set_attr("tensor_id", tensor_id); - if (is_input) { - input_ops_name_.emplace_back(tf_node->get_name()); - } - if (is_output) { - output_ops_name_.emplace_back(tf_node->get_name()); - } - } - } - - void process_kernel_subgraphs() { - auto root = graph_->get_root_subgraph(); - // CHECK_EQ(root->get_children_num(), 2) - // << "only support one input kernel and one dpu kernel"; - for (auto& kernel_subgraph : root->get_children()) { - auto device = kernel_subgraph->get_attr("device"); - if (device == "DPU" && kernel_subgraph->get_name() == kernel_) { - attach_code_to_kernel_subgraph(kernel_subgraph); - attach_parameter_to_kernel_subgraph(kernel_subgraph); - attach_input_and_output_ops_name_to_subgraph(kernel_subgraph); - attach_runner_to_kernel_subgraph(kernel_subgraph); - } - } - } - void attach_code_to_kernel_subgraph(xir::Subgraph* kernel_subgraph) { - std::vector code_buffer; - code_buffer.reserve(4096); - auto code = elf_->Code(); - const auto& nodes = elf_->Nodes(); - auto node_id = 0; - auto debug_mode = meta_data_->mode != 0; - CHECK(!debug_mode) - << "only release mode is supported. when code is attached to kernel " - "subgraph"; - - for (const auto& node : nodes) { - auto node_name = elf_->deephi_string(node->name); - if (node->type == DpuCodeNode) { - CHECK_EQ(node->codes()->cnt, 1u) - << "one DPU codde per super layer! node_name=" << node_name; - auto code_size = node->codes()->data[0].size; - auto code_offset = node->codes()->data[0].offset; - code_buffer.insert(code_buffer.end(), code + code_offset, - code + code_offset + code_size); - } - node_id++; - } - LOG_IF(INFO, ENV_PARAM(DEBUG_ELF_XIR)) - << "kernel =" << kernel_subgraph->get_name() << " kernel_=" << kernel_ - << " code size=" << code_buffer.size(); - kernel_subgraph->set_attr("mc_code", code_buffer); - } - - void attach_parameter_to_kernel_subgraph(xir::Subgraph* kernel_subgraph) { - auto parameter_size = elf_->ParameterSize(); - auto parameter = elf_->Parameter(); - kernel_subgraph->set_attr>>( - "reg_id_to_parameter_value", - {{"REG_0", {parameter, parameter + parameter_size}}}); - } - void attach_runner_to_kernel_subgraph(xir::Subgraph* kernel_subgraph) { - kernel_subgraph->set_attr>( - "runner", {{"run", "libvart-dpu-runner.so"}}); - } - void attach_input_and_output_ops_name_to_subgraph(xir::Subgraph* subgraph) { - LOG_IF(INFO, ENV_PARAM(DEBUG_ELF_XIR)) - << "kernel=" << subgraph->get_name() << " " - << "input_ops_name " << input_ops_name_ << " " - << "output_ops_name_" << output_ops_name_ << " "; - subgraph->set_attr("input_ops_name", input_ops_name_); - subgraph->set_attr("output_ops_name", output_ops_name_); - subgraph->set_attr>( - "reg_id_to_context_type", {{"REG_0", "CONST"}, {"REG_1", "DATA"}}); - subgraph->set_attr>("reg_id_to_size", - {{"REG_1", (int)io_size_}}); - subgraph->set_attr>( - "reg_id_to_hw_segment", {{"REG_0", "W0"}, {"REG_1", "D0"}}); - } - - public: - xir::Op* operator()(xir::Graph* g, xir::Op* op) { - previous_op_ = op; - graph_ = g; - auto graph_name = filename_ + ":" + elf_->get_kernel_name(); - - add_metadata(); - tensor_id_2_node_id_ = build_tensor_id_2_node_id(); - node_id_2_kernel_id_ = build_node_id_2_kernel_id(); - connect_tf_nodes(); - add_tf_nodes(); - return last_op_; - } - /* void add_subgraphs(xir::Graph* g) { - // add_tf_nodes_preload(); - // add_super_layer_preload(); - add_subgraphs(); - return; - }*/ - void process_kernel_subgraphs(xir::Graph* g) { - process_kernel_subgraphs(); - return; - } -}; - -} // namespace -namespace xir { - -void create_super_layer_subgraph(xir::Subgraph* kernel_subgraph) { - kernel_subgraph->create_children(); - std::map> superlayers; - auto ops = kernel_subgraph->get_ops(); - for (auto op : ops) { - auto super_layer = kernel_subgraph->find_op(op); - auto super_layer_id = op->get_attr("super_layer_id"); - superlayers[super_layer_id].emplace(super_layer); - } - for (auto& superlayer_id_and_subgraph : superlayers) { - auto superlayer_subgraph = - kernel_subgraph->merge_children(superlayer_id_and_subgraph.second); - superlayer_subgraph->set_name(superlayer_id_and_subgraph.first); - } -} -bool check_device_is_dpu(const std::string& kernel_id) { - return kernel_id.substr(kernel_id.size() - 3) != "_-1" && - kernel_id != "global_kernel_id"; -} - -void add_subgraphs(xir::Graph* g) { - // auto debug_mode = meta_data_->mode != 0; - auto ops = g->get_ops(); - auto root = g->get_root_subgraph(); - root->create_children(); - std::map> kernel_subgraphs; - for (auto op : ops) { - auto subgraph = root->find_op(op); - auto kernel_id = op->get_attr("kernel_id"); - kernel_subgraphs[kernel_id].emplace(subgraph); - // LOG(INFO) << "kernel id " << kernel_id; - } - // CHECK(!debug_mode) << "debug mode not supported yet."; - // CHECK_EQ(kernel_subgraphs.size(), 2u) - // << "only support one input kernel and one dpu kernel."; - for (auto& kernel_name_and_subgraphs : kernel_subgraphs) { - auto kernel_subgraph = - root->merge_children(kernel_name_and_subgraphs.second); - auto kernel_id = kernel_name_and_subgraphs.first; - if (check_device_is_dpu(kernel_id)) { - kernel_subgraph->set_attr("device", "DPU"); - // kernel_subgraph->set_name(kernel_id.substr(0, kernel_.size())); - - kernel_subgraph->set_name( - kernel_id.substr(0, kernel_id.find_last_of("_"))); - // kernel_subgraph->set_name(elf_->get_kernel_name()); - } else { - kernel_subgraph->set_attr("device", "USER"); - kernel_subgraph->set_name(kernel_id); - } - LOG_IF(INFO, ENV_PARAM(DEBUG_ELF_XIR)) - << "kernel_subgraphs.size() " << kernel_subgraphs.size() - << " " // - " kernel-name= " - << kernel_subgraph->get_name(); - create_super_layer_subgraph(kernel_subgraph); - } - return; -} - -std::unique_ptr my_elf2xir(const std::string& filename) { - auto graph_name = filename; - auto graph = xir::Graph::create(graph_name); - auto kernels = Elf::create(filename); - auto op = graph->add_op("global_input", std::string("data"), - xir::Attrs::create(), {}); - op->replace_output_tensor(xir::Tensor::create( - "global_input", {1, 1, 1, 1}, xir::DataType{xir::DataType::XINT, 8})); - op->set_attr("kernel_id", "global_kernel_id"); - op->set_attr("node_type", "global"); - op->set_attr("super_layer_id", "global_super_layer_id"); - auto objs = vector{}; - if (0) - for (auto& k : kernels) { - LOG(INFO) << "k-> " << k.first << ":" << (void*)k.second.get(); - } - for (auto& k : kernels) { - auto elf2tf_obj = elf2xir(k.second.get()); - op = elf2tf_obj(graph.get(), op); - objs.emplace_back(std::move(elf2tf_obj)); - } - - add_subgraphs(graph.get()); - for (auto& o : objs) { - o.process_kernel_subgraphs(graph.get()); - } - if (ENV_PARAM(DEBUG_ELF_XIR)) { - graph->serialize("a.xmodel"); - } - return graph; -} -} // namespace xir - -// Local Variables: -// mode:c++ -// coding: utf-8-unix -// End: diff -pruN 1.4-2/src/xir/graph/graph.cpp 2.5-1/src/xir/graph/graph.cpp --- 1.4-2/src/xir/graph/graph.cpp 2021-07-27 05:41:52.000000000 +0000 +++ 2.5-1/src/xir/graph/graph.cpp 2022-06-15 07:33:00.000000000 +0000 @@ -26,24 +26,10 @@ std::unique_ptr Graph::create(std return std::unique_ptr{static_cast(new GraphImp{name})}; } -extern int g_use_proto_version; // defined in GraphImp.cpp -static bool is_elf(const std::string& filename) { - std::string magic; - magic.resize(4); - CHECK(std::ifstream(filename).read(&magic[0], magic.size()).good()) - << "read fail! filename=" << filename; - return magic[0] == 0x7f && magic[1] == 0x45 && magic[2] == 0x4c && - magic[3] == 0x46; -} -// defined in elf2xir.cpp -extern std::unique_ptr my_elf2xir(const std::string&); std::unique_ptr Graph::deserialize(const std::string& pb_fname) { - if (!is_elf(pb_fname)) { - v2::Serialize s; - auto g = s.read(pb_fname); - return g; - } - return my_elf2xir(pb_fname); + v2::Serialize s; + auto g = s.read(pb_fname); + return g; } std::unique_ptr Graph::deserialize_from_string(const std::string& str) { diff -pruN 1.4-2/src/xir/graph/graph_imp.cpp 2.5-1/src/xir/graph/graph_imp.cpp --- 1.4-2/src/xir/graph/graph_imp.cpp 2021-07-27 05:41:52.000000000 +0000 +++ 2.5-1/src/xir/graph/graph_imp.cpp 2022-06-15 07:33:00.000000000 +0000 @@ -250,10 +250,13 @@ void GraphImp::visualize(const std::stri auto create_fig_from_dot = "dot -T" + format + " " + file_name_dot + " -o " + file_path; auto result = std::system(create_fig_from_dot.c_str()); - UNI_LOG_CHECK(result == 0, XIR_OPERATION_FAILED) << create_fig_from_dot; + UNI_LOG_CHECK(result == 0, XIR_OPERATION_FAILED) + << "command: \"" << create_fig_from_dot << "\", std::system exit(" + << result << ")"; auto rm_dot = "rm " + file_name_dot; result = std::system(rm_dot.c_str()); - UNI_LOG_CHECK(result == 0, XIR_OPERATION_FAILED) << rm_dot; + UNI_LOG_CHECK(result == 0, XIR_OPERATION_FAILED) + << "command: \"" << rm_dot << "\", std::system exit(" << result << ")"; } void GraphImp::serialize(const std::string& file_path) const { @@ -350,71 +353,4 @@ const std::string GraphImp::to_string(co << right_bracket; return out.str(); } - } // namespace xir - -/* C API implementations */ -#include "xir/xir.h" -extern "C" xir_graph_t xir_graph_create(const char* name) { - auto g = xir::Graph::create(std::string(name)); - return static_cast(g.release()); -} -extern "C" xir_graph_t xir_graph_deserialize(const char* name) { - auto g = xir::Graph::deserialize(std::string(name)); - return static_cast(g.release()); -} -extern "C" int xir_graph_destroy(xir_graph_t graph) { - auto g = static_cast(graph); - delete g; - return 0; -} -namespace xir { -class c_api { - public: - static const char* get_name(xir_graph_t graph) { - auto self = static_cast(graph); - return boost::get_property(*self->graph_, boost::graph_name).c_str(); - } -}; -}; // namespace xir -extern "C" const char* xir_graph_get_name(xir_graph_t graph) { - return xir::c_api::get_name(graph); -} - -static std::map> build_input_ops( - struct xir_graph_input_ops_t* input_ops, size_t num_of_ops) { - std::map> ret; - for (auto i = 0u; i < num_of_ops; ++i) { - auto begin = (xir::Op**)(input_ops[i].ops); - auto size = input_ops[i].num_of_ops; - ret.emplace(std::string(input_ops[i].name), - std::vector(begin, begin + size)); - } - return ret; -} - -extern "C" xir_op_t xir_graph_add_op( - xir_graph_t graph, // - const char* name, // - const char* type, // - xir_attrs_t attrs, // - struct xir_graph_input_ops_t* input_ops, // - size_t num_of_ops, // - xir_tensor_t tensor, // - xir_subgraph_t subgraph // -) { - auto self = static_cast(graph); - auto ret = - self->add_op(std::string(name), std::string(type), - std::unique_ptr(static_cast(attrs)), - build_input_ops(input_ops, num_of_ops), - static_cast(subgraph)); - ret->replace_output_tensor( - std::unique_ptr(static_cast(tensor))); - return static_cast(ret); -} - -extern "C" xir_subgraph_t xir_graph_get_root_subgraph(xir_graph_t graph) { - return static_cast( - static_cast(graph)->get_root_subgraph()); -} diff -pruN 1.4-2/src/xir/graph/graph_template_imp.cpp 2.5-1/src/xir/graph/graph_template_imp.cpp --- 1.4-2/src/xir/graph/graph_template_imp.cpp 2021-07-27 05:41:52.000000000 +0000 +++ 2.5-1/src/xir/graph/graph_template_imp.cpp 2022-06-15 07:33:00.000000000 +0000 @@ -202,10 +202,13 @@ void GraphTemplateImp::visualize(const s auto create_fig_from_dot = "dot -T" + format + " " + file_name_dot + " -o " + filename + "." + format; auto result = std::system(create_fig_from_dot.c_str()); - UNI_LOG_CHECK(result == 0, XIR_OPERATION_FAILED) << create_fig_from_dot; + UNI_LOG_CHECK(result == 0, XIR_OPERATION_FAILED) + << "command: \"" << create_fig_from_dot << "\", std::system exit(" + << result << ")"; auto rm_dot = "rm " + file_name_dot; result = std::system(rm_dot.c_str()); - UNI_LOG_CHECK(result == 0, XIR_OPERATION_FAILED) << rm_dot; + UNI_LOG_CHECK(result == 0, XIR_OPERATION_FAILED) + << "command: \"" << rm_dot << "\", std::system exit(" << result << ")"; } GraphTemplateImp::GraphType* GraphTemplateImp::get_boost_graph() { diff -pruN 1.4-2/src/xir/graph/serialize_v2.cpp 2.5-1/src/xir/graph/serialize_v2.cpp --- 1.4-2/src/xir/graph/serialize_v2.cpp 2021-07-27 05:41:52.000000000 +0000 +++ 2.5-1/src/xir/graph/serialize_v2.cpp 2022-06-15 07:33:00.000000000 +0000 @@ -15,6 +15,8 @@ */ #include "xir/graph/serialize_v2.hpp" +// must include this first to define XIR_DLLSPEC; +#include "xir/XirExport.hpp" #include #include @@ -23,6 +25,7 @@ #include "config.hpp" #include "graph_proto_v2.pb.h" + #include "xir/attrs/attr_def.hpp" #include "xir/attrs/attrs.hpp" #include "xir/graph/graph.hpp" @@ -497,7 +500,7 @@ xir::AttrDef convert_t Serialize::read_from_s serial_v2::Graph pb_graph; if (!pb_graph.ParseFromString(str)) { UNI_LOG_FATAL(XIR_READ_PB_FAILURE) - << "fail to generate pb struct from string."; + << "fail to generate pb struct from string."; } return convert_from_pb_type_tp_cpp_type( - *(static_cast(&pb_graph))); + *(static_cast(&pb_graph))); } void Serialize::write(const Graph* graph, const string& pb_fname) { @@ -726,8 +729,7 @@ void Serialize::write_to_string(const Gr serial_v2::Graph pb_graph = convert_from_cpp_type_to_pb(g); g.release(); if (!pb_graph.SerializeToString(str)) { - UNI_LOG_FATAL(XIR_WRITE_PB_FAILURE) - << "fail to parse pb struct to string."; + UNI_LOG_FATAL(XIR_WRITE_PB_FAILURE) << "fail to parse pb struct to string."; } } diff -pruN 1.4-2/src/xir/graph/subgraph_imp.cpp 2.5-1/src/xir/graph/subgraph_imp.cpp --- 1.4-2/src/xir/graph/subgraph_imp.cpp 2021-07-27 05:41:52.000000000 +0000 +++ 2.5-1/src/xir/graph/subgraph_imp.cpp 2022-06-15 07:33:00.000000000 +0000 @@ -88,6 +88,22 @@ const std::set SubgraphIm return ret; } +std::vector SubgraphImp::get_sorted_input_tensors( + const std::function& compare) { + return internal::cast_from_const_vector( + static_cast(*this).get_sorted_input_tensors(compare)); +} + +const std::vector SubgraphImp::get_sorted_input_tensors( + const std::function& compare) const { + auto ret = std::vector(); + auto ret_s = internal::cast_from_const_set( + static_cast(*this).get_input_tensors()); + ret.assign(ret_s.begin(), ret_s.end()); + sort(ret.begin(), ret.end(), compare); + return internal::cast_to_const_vector(ret); +} + std::set SubgraphImp::get_output_tensors() { return internal::cast_from_const_set( static_cast(*this).get_output_tensors()); @@ -110,6 +126,23 @@ const std::set SubgraphIm return ret; } +std::vector SubgraphImp::get_sorted_output_tensors( + const std::function& compare) { + return internal::cast_from_const_vector( + static_cast(*this).get_sorted_output_tensors( + compare)); +} + +const std::vector SubgraphImp::get_sorted_output_tensors( + const std::function& compare) const { + auto ret = std::vector(); + auto ret_s = internal::cast_from_const_set( + static_cast(*this).get_output_tensors()); + ret.assign(ret_s.begin(), ret_s.end()); + sort(ret.begin(), ret.end(), compare); + return internal::cast_to_const_vector(ret); +} + std::int32_t SubgraphImp::count_op_( const std::set& op_types) const { std::int32_t ret = 0; @@ -722,51 +755,4 @@ std::vector> return *result; } - } // namespace xir - -/* C API implementations */ -#include "xir/xir.h" -namespace xir { -class c_api { - public: - static const char* subgraph_get_name(xir_subgraph_t subgraph) { - auto self = static_cast(subgraph); - return self->name_.c_str(); - } - static xir_subgraph_t subgraph_get_child(xir_subgraph_t subgraph, - int32_t idx) { - auto self = static_cast(subgraph); - auto child = self->children_.begin(); - for (int i = 0; i < idx; i++) { - child++; - } - return static_cast((*child).get()); - } - - static void subgraph_children_topological_sort(xir_subgraph_t subgraph, - xir_subgraph_t children[]) { - auto self = static_cast(subgraph); - auto ret = self->children_topological_sort(); - int i = 0; - for (auto child : ret) { - children[i++] = static_cast(child); - } - } -}; -}; // namespace xir -extern "C" const char* xir_subgraph_get_name(xir_subgraph_t subgraph) { - return xir::c_api::subgraph_get_name(subgraph); -} -extern "C" int32_t xir_subgraph_get_children_num(xir_subgraph_t subgraph) { - return static_cast(subgraph)->get_children_num(); -} -extern "C" xir_subgraph_t xir_subgraph_get_child(xir_subgraph_t subgraph, - int32_t idx) { - return xir::c_api::subgraph_get_child(subgraph, idx); -} - -extern "C" void xir_subgraph_children_topological_sort( - xir_subgraph_t subgraph, xir_subgraph_t children[]) { - return xir::c_api::subgraph_children_topological_sort(subgraph, children); -} diff -pruN 1.4-2/src/xir/graph/subgraph_imp.hpp 2.5-1/src/xir/graph/subgraph_imp.hpp --- 1.4-2/src/xir/graph/subgraph_imp.hpp 2021-07-27 05:41:52.000000000 +0000 +++ 2.5-1/src/xir/graph/subgraph_imp.hpp 2022-06-15 07:33:00.000000000 +0000 @@ -98,9 +98,19 @@ class SubgraphImp : public Subgraph { std::set get_input_tensors() override; const std::set get_input_tensors() const override; + std::vector get_sorted_input_tensors( + const std::function& compare) override; + const std::vector get_sorted_input_tensors( + const std::function& compare) const override; + std::set get_output_tensors() override; const std::set get_output_tensors() const override; + std::vector get_sorted_output_tensors( + const std::function& compare) override; + const std::vector get_sorted_output_tensors( + const std::function& compare) const override; + bool has_op(const std::string& op_name) const override; bool has_op(const Op* op) const override; @@ -199,7 +209,6 @@ class SubgraphImp : public Subgraph { std::set ops_; std::unique_ptr attrs_; - friend class c_api; }; } // namespace xir diff -pruN 1.4-2/src/xir/op/built_in_ops.cpp 2.5-1/src/xir/op/built_in_ops.cpp --- 1.4-2/src/xir/op/built_in_ops.cpp 2021-07-27 05:41:52.000000000 +0000 +++ 2.5-1/src/xir/op/built_in_ops.cpp 2022-06-15 07:33:00.000000000 +0000 @@ -13,39 +13,32 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "xir/op/built_in_ops.hpp" - #include +#include "xir/op/built_in_ops.hpp" #include "xir/op/shape_inference.hpp" using namespace xir; - +using std::string; #define XIR_REGISTER_BUILT_IN_OP(OP) \ - static BuiltInOPsRegisterer BUILT_IN_OPDEFS_##OP(OP); + static BuiltInOPsRegister BUILT_IN_OPDEFS_##OP(OP); -class BuiltInOPsRegistry { +class BuiltInOPsRegister { public: - static std::vector BUILT_IN_OPS_; - BuiltInOPsRegistry() = default; + BuiltInOPsRegister(const xir::OpDef& def) { add_op_def(def); } + static void add_op_def(const xir::OpDef& def) { BUILT_IN_OPS_.push_back(std::move(def)); } -}; -std::vector BuiltInOPsRegistry::BUILT_IN_OPS_ = - std::vector{}; - -class BuiltInOPsRegisterer { - public: - BuiltInOPsRegisterer(const xir::OpDef& def) { - BuiltInOPsRegistry::add_op_def(def); - } + static std::vector BUILT_IN_OPS_; }; +std::vector BuiltInOPsRegister::BUILT_IN_OPS_ = {}; + void register_built_in_ops(xir::OpDefFactory* self) { - std::for_each(BuiltInOPsRegistry::BUILT_IN_OPS_.begin(), - BuiltInOPsRegistry::BUILT_IN_OPS_.end(), + std::for_each(BuiltInOPsRegister::BUILT_IN_OPS_.begin(), + BuiltInOPsRegister::BUILT_IN_OPS_.end(), [self](const xir::OpDef& def) { self->register_h(def); }); } @@ -170,7 +163,7 @@ std::function ConvOpD "The value must be: `{dilation_width, dilation_height}`, " "The dilation in the batch or depth are 1 in default.", {1, 1}); - auto pad_mode = xir::AttrDefBuilder::build( + auto pad_mode = xir::AttrDefBuilder::build( "pad_mode", AttrDef::REQUIRED, "`Datatype`: `string`\n\n" "We support 4 padding mode: `FLOOR, CEIL, SAME, VALID`. " @@ -365,6 +358,113 @@ XIR_REGISTER_BUILT_IN_OP(depthwiseconv2d XIR_REGISTER_BUILT_IN_OP(transposed_conv2d); XIR_REGISTER_BUILT_IN_OP(transposed_depthwise_conv2d); +std::function Conv3dOpDefGenerator(xir::DataType::Type T) { + return [=](xir::OpDef& op_def) { + auto input = xir::OpArgDef{"input", OpArgDef::REQUIRED, T, + "An input tensor with shape " + "`[batch, in_height, in_width, depth, in_channels]`."}; + auto weights = xir::OpArgDef{ + "weights", OpArgDef::REQUIRED, T, + "A filter tensor with shape " + "`[output_channels, kernel_height, kernel_width, kernel_depth, in_channels]`."}; + auto bias = xir::OpArgDef{"bias", OpArgDef::OPTIONAL, T, + "A bias tensor with shape " + "`[output_channels]`."}; + auto kernel = xir::AttrDefBuilder>::build( + "kernel", AttrDef::REQUIRED, 3, + "`Datatype`: `vector`\n\n" + "The kernel sizes of the filter. " + "The value must be: `{kernel_width, kernel_height, kernel_depth}`."); + auto stride = xir::AttrDefBuilder>::build( + "stride", AttrDef::REQUIRED, 3, + "`Datatype`: `vector`\n\n" + "The strides of the filter. " + "The value must be: `{stride_width, stride_height, stride_depth}`."); + auto dilation = xir::AttrDefBuilder>::build( + "dilation", AttrDef::OPTIONAL, 3, + "`Datatype`: `vector`\n\n" + "The dilation of the filter. " + "The value must be: `{dilation_width, dilation_height, dilation_depth}`, " + "The dilation in the batch or depth are 1 in default.", + {1, 1, 1}); + auto pad_mode = xir::AttrDefBuilder::build( + "pad_mode", AttrDef::REQUIRED, + "`Datatype`: `string`\n\n" + "We support 4 padding mode: `FLOOR, CEIL, SAME, VALID`. " + "For example, when you parsing models from other frameworks, " + "`caffe, pytorch->\"FLOOR\", tensorflow->\"SAME\" or \"VALID\"`."); + auto pad = xir::AttrDefBuilder>::build( + "pad", AttrDef::OPTIONAL, 6, + "`Datatype`: `vector`\n\n" + "The padding sizes of input feature maps. " + "The value must be `{left, right, top, bottom, near, far}`.\n\n" + "For transposed convolutions, the padding here denotes the " + "`{kernel_size - 1 - actual_padding}`." + "This is an optional attribute, when the pad_mode is SAME or VALID, " + "you don't need to specify this attribute.", + {0, 0, 0, 0, 0, 0}); + op_def.add_input_arg(input) + .add_input_arg(weights) + .add_input_arg(bias) + .add_attr(kernel) + .add_attr(stride) + .add_attr(dilation) + .add_attr(pad_mode) + .add_attr(pad); + }; +} + +auto conv3d = xir::OpDef("conv3d") + .inherit_from(Conv3dOpDefGenerator(xir::DataType::FLOAT)) + .set_annotation( + "3D convolution.\n\n") + .set_shape_infer(xir::shape_infer_conv3d); + +auto transposed_conv3d = xir::OpDef("transposed-conv3d") + .inherit_from(Conv3dOpDefGenerator(xir::DataType::FLOAT)) + .set_annotation( + "Transposed 3D convolution.\n\n") + .set_shape_infer(xir::shape_infer_transposed_conv3d); + +auto depthwise_conv3d = + xir::OpDef("depthwise-conv3d") + .inherit_from(Conv3dOpDefGenerator(xir::DataType::FLOAT)) + .set_annotation( + "Depth-wise 3D convolution.\n\n") + .set_shape_infer(xir::shape_infer_depthwise_conv3d) + .add_constraint([](xir::Op* op) { + auto weights = op->get_input_tensor("weights"); + auto w_shape = weights->get_shape(); + auto in = op->get_input_tensor("input"); + auto in_shape = in->get_shape(); + UNI_LOG_CHECK(w_shape[4] == in_shape[4], XIR_INVALID_ARG_OCCUR) + << "The channel of weights should be equal to the channel of " + "input in " + "depthwise conv3d"; + }); + +auto transposed_depthwise_conv3d = + xir::OpDef("transposed-depthwise-conv3d") + .inherit_from(Conv3dOpDefGenerator(xir::DataType::FLOAT)) + .set_annotation( + "Transposed depth-wise 3D convolution.\n\n") + .set_shape_infer(xir::shape_infer_transposed_depthwise_conv3d) + .add_constraint([](xir::Op* op) { + auto weights = op->get_input_tensor("weights"); + auto w_shape = weights->get_shape(); + auto in = op->get_input_tensor("input"); + auto in_shape = in->get_shape(); + UNI_LOG_CHECK(w_shape[4] == in_shape[4], XIR_INVALID_ARG_OCCUR) + << "The channel of weights should be equal to the channel of " + "input in " + "depthwise conv3d"; + }); + +XIR_REGISTER_BUILT_IN_OP(conv3d); +XIR_REGISTER_BUILT_IN_OP(depthwise_conv3d); +XIR_REGISTER_BUILT_IN_OP(transposed_conv3d); +XIR_REGISTER_BUILT_IN_OP(transposed_depthwise_conv3d); + std::function FixedConvOpDefGenerator( xir::DataType::Type T) { return [=](xir::OpDef& op_def) { @@ -401,9 +501,7 @@ std::function FixedCo "The padding sizes of input feature maps. " "The value must be `{left, right, top, bottom}`.\n\n" "For transposed convolutions, the padding here denotes the " - "`{kernel_size - 1 - actual_padding}`." - "This is an optional attribute, when the pad_mode is SAME or VALID, " - "you don't need to specify this attribute.", + "`{kernel_size - 1 - actual_padding}`.", {0, 0, 0, 0}); auto nonlinear = xir::AttrDefBuilder::build( "nonlinear", AttrDef::OPTIONAL, @@ -488,6 +586,129 @@ XIR_REGISTER_BUILT_IN_OP(depthwise_conv2 XIR_REGISTER_BUILT_IN_OP(transposed_conv2d_fix); XIR_REGISTER_BUILT_IN_OP(transposed_depthwise_conv2d_fix); +std::function FixedConv3dOpDefGenerator( + xir::DataType::Type T) { + return [=](xir::OpDef& op_def) { + auto input = xir::OpArgDef{"input", OpArgDef::REQUIRED, T, + "An input tensor with shape " + "`[batch, in_height, in_width, in_depth, in_channels]`."}; + auto weights = xir::OpArgDef{ + "weights", OpArgDef::REQUIRED, T, + "A filter tensor with shape " + "`[output_channels, kernel_height, kernel_width, kernel_depth, in_channels]`."}; + auto bias = xir::OpArgDef{"bias", OpArgDef::OPTIONAL, T, + "A bias tensor with shape " + "`[output_channels]`."}; + auto kernel = xir::AttrDefBuilder>::build( + "kernel", AttrDef::REQUIRED, 3, + "`Datatype`: `vector`\n\n" + "The kernel sizes of the filter. " + "The value must be: `{kernel_width, kernel_height}`."); + auto stride = xir::AttrDefBuilder>::build( + "stride", AttrDef::REQUIRED, 3, + "`Datatype`: `vector`\n\n" + "The strides of the filter. " + "The value must be: `{stride_width, stride_height}`."); + auto dilation = xir::AttrDefBuilder>::build( + "dilation", AttrDef::OPTIONAL, 3, + "`Datatype`: `vector`\n\n" + "The dilation of the filter. " + "The value must be: `{dilation_width, dilation_height}`, " + "The dilation in the batch or depth are 1 in default.", + {1, 1, 1}); + auto pad = xir::AttrDefBuilder>::build( + "pad", AttrDef::OPTIONAL, 6, + "`Datatype`: `vector`\n\n" + "The padding sizes of input feature maps. " + "The value must be `{left, right, top, bottom}`.\n\n" + "For transposed convolutions, the padding here denotes the " + "`{kernel_size - 1 - actual_padding}`." + "This is an optional attribute, when the pad_mode is SAME or VALID, " + "you don't need to specify this attribute.", + {0, 0, 0, 0, 0, 0}); + auto nonlinear = xir::AttrDefBuilder::build( + "nonlinear", AttrDef::OPTIONAL, + "`Datatype`: `string`\n\n" + "nonlinear type, \"NONE\", \"RELU\", \"PRELU\", " + "\"LEAKYRELU\",\"RELU6\",\"HSIGMOID\",\"HSWISH\".", + ""); + op_def.add_input_arg(input) + .add_input_arg(weights) + .add_input_arg(bias) + .add_attr(kernel) + .add_attr(stride) + .add_attr(dilation) + .add_attr(pad) + .add_attr(nonlinear); + }; +} + +auto conv3d_fix = + xir::OpDef("conv3d-fix") + .inherit_from(FixedConv3dOpDefGenerator(xir::DataType::XINT)) + .add_attr(xir::AttrDefBuilder::build("hsigmoid_in", + AttrDef::OPTIONAL, + "`Datatype`: `int`\n\n" + "fix_point of hsigmoid", + -128)) + .add_attr(xir::AttrDefBuilder::build("shift_hsigmoid", + AttrDef::OPTIONAL, + "`Datatype`: `int`\n\n" + "shift value after hsigmoid", + -128)) + .add_attr(xir::AttrDefBuilder::build("shift_hswish", + AttrDef::OPTIONAL, + "`Datatype`: `int`\n\n" + "shift value after hswish", + -128)) + .add_constraint([](xir::Op* op) { + if (op->has_attr("nonlinear")) { + if (op->get_attr("nonlinear") == "HSIGMOID" || + op->get_attr("nonlinear") == "HSWISH") { + UNI_LOG_CHECK(op->get_attr("hsigmoid_in") != -128 && + op->get_attr("shift_hsigmoid") != -128, + XIR_INVALID_ARG_OCCUR) + << "the activation type of conv3d-fix is " + << op->get_attr("nonlinear") + << " but you do not set the shift value for this operation."; + if (op->get_attr("nonlinear") == "HSWISH") + UNI_LOG_CHECK(op->get_attr("shift_hswish") != -128, + XIR_INVALID_ARG_OCCUR) + << "the activation type of conv3d-fix is " + << op->get_attr("nonlinear") + << " but you do not set the shift_hswish for this " + "operation."; + UNI_LOG_CHECK( + op->has_attr("shift_bias") && op->has_attr("shift_cut"), + XIR_INVALID_ARG_OCCUR) + << "the activation type of conv3d-fix is " + << op->get_attr("nonlinear") + << " you need to set shift_bias and shift_cut for it.."; + } + } + }) + .set_shape_infer(xir::shape_infer_conv3d_fix); + +auto depthwise_conv3d_fix = + xir::OpDef("depthwise-conv3d-fix") + .inherit_from(FixedConv3dOpDefGenerator(xir::DataType::XINT)) + .set_shape_infer(xir::shape_infer_depthwise_conv3d_fix); + +auto transposed_conv3d_fix = + xir::OpDef("transposed-conv3d-fix") + .inherit_from(FixedConv3dOpDefGenerator(xir::DataType::XINT)) + .set_shape_infer(xir::shape_infer_transposed_conv3d_fix); + +auto transposed_depthwise_conv3d_fix = + xir::OpDef("transposed-depthwise-conv3d-fix") + .inherit_from(FixedConv3dOpDefGenerator(xir::DataType::XINT)) + .set_shape_infer(xir::shape_infer_transposed_depthwise_conv3d_fix); + +XIR_REGISTER_BUILT_IN_OP(conv3d_fix); +XIR_REGISTER_BUILT_IN_OP(depthwise_conv3d_fix); +XIR_REGISTER_BUILT_IN_OP(transposed_conv3d_fix); +XIR_REGISTER_BUILT_IN_OP(transposed_depthwise_conv3d_fix); + std::function Pool2dOpDefGenerator(xir::DataType::Type T) { return [=](xir::OpDef& op_def) { auto input = xir::OpArgDef{"input", OpArgDef::REQUIRED, T, @@ -514,9 +735,7 @@ std::function Pool2dO "pad", AttrDef::OPTIONAL, 4, "`Datatype`: `vector`\n\n" "The padding sizes of input feature maps. " - "The value must be `{left, right, top, bottom}`. " - "This is an optional attribute, when the pad_mode is SAME or VALID, " - "you don't need to specify this attribute.", + "The value must be `{left, right, top, bottom}`.", {0, 0, 0, 0}); auto global = xir::AttrDefBuilder::build( "global", AttrDef::OPTIONAL, @@ -1198,7 +1417,7 @@ std::function ResizeO .add_attr( xir::AttrDefBuilder::build("mode", AttrDef::REQUIRED, "`Datatype`: `string`\n\n" - "NEAREST or BILINEAR")) + "NEAREST, BILINEAR or TRILINEAR")) .add_attr(xir::AttrDefBuilder::build( "align_corners", AttrDef::OPTIONAL, "`Datatype`: `bool`\n\n" @@ -1217,15 +1436,15 @@ auto resize = xir::OpDef("resize") .inherit_from(ResizeOpDefGenerator(xir::DataType::FLOAT)) .add_attr(xir::AttrDefBuilder>::build( - "scale", AttrDef::OPTIONAL, 2, + "scale", AttrDef::OPTIONAL, 0, "`Datatype`: `vector`\n\n" "Constant values denotes the scale to resize the input. " "scale = out / in", - {1, 1})) + {})) .set_annotation( "Operator resize the feature maps. For example, if the input " "is an image, and the shape of this image is [h, w, c], after " - "resize, " + "2d resize, " "the shape of the output image is [oh, ow, c].\n\n" " scale = (align_corners && out > 1)\n" " ? (in - 1) / (out - 1)\n" @@ -1256,16 +1475,30 @@ auto resize = " (image[bottom, left, c] * (1 - x_lerp) +\n" " image[bottom, right, c] * x_lerp) * " "y_lerp\n") + .add_constraint([](xir::Op* op) { + auto mode = op->get_attr("mode"); + auto in = op->get_input_tensor("input"); + auto in_shape = in->get_shape(); + if (mode == "NEAREST" || mode == "BILINEAR") { + UNI_LOG_CHECK(in_shape.size() == 4, XIR_INVALID_ARG_OCCUR) + << "We only support NEAREST and BILINEAR resize for 4-D " + "feature maps."; + } + if (mode == "TRILINEAR") { + UNI_LOG_CHECK(in_shape.size() == 5, XIR_INVALID_ARG_OCCUR) + << "We only support TRILINEAR resize for 5-D feature maps."; + } + }) .set_shape_infer(xir::shape_infer_resize); auto upsample_fix = xir::OpDef("upsample-fix") .inherit_from(ResizeOpDefGenerator(xir::DataType::XINT)) - .add_attr(xir::AttrDefBuilder>::build( + .add_attr(xir::AttrDefBuilder>::build( "scale", // AttrDef::REQUIRED, // 2, // - "`DataType` : `std::vector` " // + "`DataType` : `std::vector` " // "{scale_w, scale_h}" // )) .set_shape_infer(xir::shape_infer_upsample_fix); @@ -1273,11 +1506,11 @@ auto upsample_fix = auto downsample_fix = xir::OpDef("downsample-fix") .inherit_from(ResizeOpDefGenerator(xir::DataType::XINT)) - .add_attr(xir::AttrDefBuilder>::build( + .add_attr(xir::AttrDefBuilder>::build( "scale", // AttrDef::REQUIRED, // 2, // - "`DataType` : `std::vector` " // + "`DataType` : `std::vector` " // "{scale_w, scale_h}" // )) .set_shape_infer(xir::shape_infer_downsample_fix); @@ -1374,6 +1607,35 @@ auto reorg_fix = xir::OpDef("reorg-fix") XIR_REGISTER_BUILT_IN_OP(reorg); XIR_REGISTER_BUILT_IN_OP(reorg_fix); +std::function PixelShuffleOpDefGenerator(xir::DataType::Type T) { + return [=](xir::OpDef& op_def) { + op_def + .add_input_arg( + xir::OpArgDef{"input", OpArgDef::REQUIRED, xir::DataType::FLOAT, + "`[batch, in_height, in_width, in_channels]`."}) + .add_attr( + xir::AttrDefBuilder::build("scale", AttrDef::REQUIRED, + "`Datatype`: `int`\n\n" + "scale for PixelShuffle")) + .add_attr(xir::AttrDefBuilder::build("upscale", AttrDef::REQUIRED, + "`Datatype`: `bool`\n\n" + "upscale or downscale PixelShuffle.")); + }; +} + +auto pixel_shuffle = xir::OpDef("pixel-shuffle") + .inherit_from(PixelShuffleOpDefGenerator(xir::DataType::FLOAT)) + .set_annotation( + "https://pytorch.org/docs/stable/generated/torch.nn.PixelShuffle.html") + .set_shape_infer(xir::shape_infer_pixel_shuffle); + +auto pixel_shuffle_fix = xir::OpDef("pixel-shuffle-fix") + .inherit_from(ReorgOpDefGenerator(xir::DataType::XINT)) + .set_shape_infer(xir::shape_infer_pixel_shuffle_fix); + +XIR_REGISTER_BUILT_IN_OP(pixel_shuffle); +XIR_REGISTER_BUILT_IN_OP(pixel_shuffle_fix); + auto softmax = xir::OpDef("softmax") .add_input_arg(xir::OpArgDef{"input", OpArgDef::REQUIRED, @@ -1442,22 +1704,23 @@ std::function PadOpDe }; } -auto pad = - xir::OpDef("pad") - .inherit_from(PadOpDefGenerator(xir::DataType::FLOAT)) - .add_attr(xir::AttrDefBuilder>::build( - "constant_values", AttrDef::REQUIRED, 0, - "`Datatype`: `vector`\n\n" - "the value set into the padded locations, 2 * len(paddings)")) - .set_shape_infer(xir::shape_infer_pad); +auto pad = xir::OpDef("pad") + .inherit_from(PadOpDefGenerator(xir::DataType::FLOAT)) + .add_attr(xir::AttrDefBuilder>::build( + "constant_values", AttrDef::OPTIONAL, 0, + "`Datatype`: `vector`\n\n" + "the value set into the padded locations, 2 * len(paddings)", + {})) + .set_shape_infer(xir::shape_infer_pad); auto pad_fix = xir::OpDef("pad-fix") .inherit_from(PadOpDefGenerator(xir::DataType::XINT)) .add_attr(xir::AttrDefBuilder>::build( - "constant_values", AttrDef::REQUIRED, 0, + "constant_values", AttrDef::OPTIONAL, 0, "`Datatype`: `vector`\n\n" - "the value set into the padded locations, 2 * len(paddings)")) + "the value set into the padded locations, 2 * len(paddings)", + {})) .set_shape_infer(xir::shape_infer_pad_fix); XIR_REGISTER_BUILT_IN_OP(pad); diff -pruN 1.4-2/src/xir/op/built_in_ops.hpp 2.5-1/src/xir/op/built_in_ops.hpp --- 1.4-2/src/xir/op/built_in_ops.hpp 2021-07-27 05:41:52.000000000 +0000 +++ 2.5-1/src/xir/op/built_in_ops.hpp 2022-06-15 07:33:00.000000000 +0000 @@ -19,6 +19,6 @@ #include "xir/op/op_def.hpp" -class BuiltInOPsRegisterer; +class BuiltInOPsRegister; void register_built_in_ops(xir::OpDefFactory* self); diff -pruN 1.4-2/src/xir/op/op_def_factory_imp.cpp 2.5-1/src/xir/op/op_def_factory_imp.cpp --- 1.4-2/src/xir/op/op_def_factory_imp.cpp 2021-07-27 05:41:52.000000000 +0000 +++ 2.5-1/src/xir/op/op_def_factory_imp.cpp 2022-06-15 07:33:00.000000000 +0000 @@ -14,19 +14,18 @@ * limitations under the License. */ -#include "xir/op/op_def_factory_imp.hpp" -#include "xir/attrs/attrs.hpp" -#include "xir/op/built_in_ops.hpp" - -#include "UniLog/UniLog.hpp" - -#include #include #include +#include #include +#include "xir/util/dynamic_load.hpp" +#include "UniLog/UniLog.hpp" +#include "xir/attrs/attrs.hpp" +#include "xir/op/built_in_ops.hpp" +#include "xir/op/op_def_factory_imp.hpp" namespace xir { - +using namespace std; static vector str_split(const string& str) { auto ret = vector{}; istringstream iss(str); @@ -52,7 +51,9 @@ static void load_ops_library(OpDefFactor reg_func(self); } -const OpDefFactoryImp* op_def_factory() { +std::mutex factory_mutex; +OpDefFactoryImp* op_def_factory() { + std::lock_guard gaurd(factory_mutex); static unique_ptr self; if (!self) { self = make_unique(); diff -pruN 1.4-2/src/xir/op/op_def_factory_imp.hpp 2.5-1/src/xir/op/op_def_factory_imp.hpp --- 1.4-2/src/xir/op/op_def_factory_imp.hpp 2021-07-27 05:41:52.000000000 +0000 +++ 2.5-1/src/xir/op/op_def_factory_imp.hpp 2022-06-15 07:33:00.000000000 +0000 @@ -32,6 +32,7 @@ class OpDefFactoryImp : public OpDefFact std::unordered_map store_; }; -const OpDefFactoryImp* op_def_factory(); +// const OpDefFactoryImp* op_def_factory(); +OpDefFactoryImp* op_def_factory(); } // namespace xir diff -pruN 1.4-2/src/xir/op/op_imp.cpp 2.5-1/src/xir/op/op_imp.cpp --- 1.4-2/src/xir/op/op_imp.cpp 2021-07-27 05:41:52.000000000 +0000 +++ 2.5-1/src/xir/op/op_imp.cpp 2022-06-15 07:33:00.000000000 +0000 @@ -15,6 +15,7 @@ */ #include "xir/op/op_imp.hpp" +#include "xir/op/shape_inference.hpp" #include "xir/util/internal_util.hpp" #include "xir/util/tool_function.hpp" @@ -47,12 +48,17 @@ OpImp::OpImp(GraphImp::VertexD vd, const const std::map>& input_ops, std::unique_ptr output_tensor, GraphImp* graph, const DataType& output_data_type) - : vd_{vd}, - to_be_removed_{false}, - name_{name}, - type_{type}, - def_{op_def_factory()->create(type)}, + : vd_{vd}, // + to_be_removed_{false}, // + name_{name}, // + type_{type}, // graph_{graph} { + auto build_in_ops = op_def_factory()->get_registered_ops(); + if (std::find(build_in_ops.begin(), build_in_ops.end(), type) == + build_in_ops.end()) { + xir::register_customized_operator_definition(name, type); + } + def_ = op_def_factory()->create(type); set_attrs(std::move(attrs)); std::for_each(def_->input_args().begin(), def_->input_args().end(), @@ -339,7 +345,7 @@ std::unique_ptr OpImp::create_ou const DataType& output_data_type) { std::vector output_tensor_shape{1}; DataType data_type; - if (!this->input_ops_.size()) { + if ((!this->input_ops_.size()) || (!this->get_input_num("input"))) { UNI_LOG_CHECK( this->attrs_->has_attr("shape") && this->attrs_->has_attr("data_type"), XIR_INVALID_ATTR_OCCUR); diff -pruN 1.4-2/src/xir/op/shape_inference.cpp 2.5-1/src/xir/op/shape_inference.cpp --- 1.4-2/src/xir/op/shape_inference.cpp 2021-07-27 05:41:52.000000000 +0000 +++ 2.5-1/src/xir/op/shape_inference.cpp 2022-06-15 07:33:00.000000000 +0000 @@ -28,7 +28,7 @@ #include "xir/util/tool_function.hpp" namespace xir { - +using namespace std; void shape_infer_unsupported(xir::Op* cur) { UNI_LOG_FATAL(XIR_INVALID_ARG_OCCUR) << "\"" << cur->get_name() << "\" is a \"" << cur->get_type() @@ -100,6 +100,68 @@ std::vector cal_out(std::v return {oh, ow}; } +std::vector cal_out_3d(std::vector in_shape, + std::vector padding, + std::vector kernel, + std::vector stride, + std::vector dilation, + std::string pad_mode) { + std::int32_t ow = 0, oh = 0, od = 0; + if (pad_mode == "FLOOR") { + oh = std::floor(1.0 * + (in_shape[1] + padding[2] + padding[3] - + ((kernel[1] - 1) * dilation[1] + 1) + stride[1]) / + stride[1]); + ow = std::floor(1.0 * + (in_shape[2] + padding[0] + padding[1] - + ((kernel[0] - 1) * dilation[0] + 1) + stride[0]) / + stride[0]); + od = std::floor(1.0 * + (in_shape[3] + padding[4] + padding[5] - + ((kernel[2] - 1) * dilation[2] + 1) + stride[2]) / + stride[2]); + + } else if (pad_mode == "CEIL") { + oh = std::ceil(1.0 * + (in_shape[1] + padding[2] + padding[3] - + ((kernel[1] - 1) * dilation[1] + 1) + stride[1]) / + stride[1]); + ow = std::ceil(1.0 * + (in_shape[2] + padding[0] + padding[1] - + ((kernel[0] - 1) * dilation[0] + 1) + stride[0]) / + stride[0]); + od = std::ceil(1.0 * + (in_shape[3] + padding[4] + padding[5] - + ((kernel[2] - 1) * dilation[2] + 1) + stride[2]) / + stride[2]); + } else if (pad_mode == "SAME") { + oh = std::ceil(1.0 * (in_shape[1] + padding[2] + padding[3]) / stride[1]); + ow = std::ceil(1.0 * (in_shape[2] + padding[0] + padding[1]) / stride[0]); + od = std::ceil(1.0 * (in_shape[3] + padding[4] + padding[5]) / stride[2]); + } else if (pad_mode == "VALID") { + oh = std::ceil(1.0 * + (in_shape[1] + padding[2] + padding[3] - + (kernel[1] - 1) * dilation[1]) / + stride[1]); + ow = std::ceil(1.0 * + (in_shape[2] + padding[0] + padding[1] - + (kernel[0] - 1) * dilation[0]) / + stride[0]); + od = std::ceil(1.0 * + (in_shape[3] + padding[4] + padding[5] - + (kernel[2] - 1) * dilation[2]) / + stride[2]); + } else { + UNI_LOG_CHECK(pad_mode == "FLOOR" || pad_mode == "CEIL" || + pad_mode == "SAME" || pad_mode == "VALID", + XIR_INVALID_ARG_OCCUR) + << "pad_mode here is " << pad_mode + << ", but it should be one of \"FLOOR\"," + "\"CEIL\", \"SAME\" or \"VALID\"."; + } + return {oh, ow, od}; +} + template std::vector read_data_in_attr(xir::Op* op) { auto data = op->get_attr>("data"); @@ -364,6 +426,228 @@ void shape_infer_transposed_depthwise_co cur->replace_output_tensor(std::move(output_tensor)); } +void shape_infer_conv3d(xir::Op* cur) { + auto in = cur->get_input_tensor("input"); + auto in_shape = in->get_shape(); + UNI_LOG_CHECK(in_shape.size() == 5, XIR_INVALID_ARG_OCCUR) + << "The number of dimension of input here is " << in_shape.size() + << ", but the number of dimension should be 5."; + auto out = cur->get_output_tensor(); + auto weights = cur->get_input_tensor("weights"); + auto w_shape = weights->get_shape(); + UNI_LOG_CHECK(w_shape.size() == 5, XIR_INVALID_ARG_OCCUR) + << "The number of dimension of weights here is " << w_shape.size() + << ", but the number of dimension should be 5."; + auto attrs = cur->get_attrs(); + auto kernel = attrs->get_attr>("kernel"); + auto stride = attrs->get_attr>("stride"); + std::vector dilation = {1, 1, 1}; + if (attrs->has_attr("dilation")) { + auto tmp = attrs->get_attr>("dilation"); + std::transform(tmp.begin(), tmp.end(), dilation.begin(), + [](auto t) { return t; }); + } + auto pad_mode = attrs->get_attr("pad_mode"); + std::vector padding = {0, 0, 0, 0, 0, 0}; + if (attrs->has_attr("pad")) { + auto tmp = attrs->get_attr>("pad"); + UNI_LOG_CHECK(tmp.size() == 6, XIR_INVALID_ARG_OCCUR) + << "The number of dimension of paddings here is " << tmp.size() + << ", but the number of dimension should be 6."; + std::transform(tmp.begin(), tmp.end(), padding.begin(), + [](auto i) { return i; }); + } + UNI_LOG_CHECK(w_shape[4] == in_shape[4], XIR_INVALID_ARG_OCCUR) + << "The channel of weights and the channel of input feature maps should be " + "the same." + << " But they are " << w_shape[4] << " and " << in_shape[4] << " now."; + auto oh_ow_od = cal_out_3d(in_shape, padding, kernel, stride, dilation, pad_mode); + std::vector new_out_shape = {in->get_shape().at(0), oh_ow_od[0], + oh_ow_od[1], oh_ow_od[2], w_shape[0]}; + auto output_tensor = + xir::Tensor::create(out->get_name(), new_out_shape, out->get_data_type()); + output_tensor->set_attrs(out->get_attrs()); + cur->replace_output_tensor(std::move(output_tensor)); +} + +void shape_infer_depthwise_conv3d(xir::Op* cur) { + auto in = cur->get_input_tensor("input"); + auto in_shape = in->get_shape(); + UNI_LOG_CHECK(in_shape.size() == 5, XIR_INVALID_ARG_OCCUR) + << "The number of dimension of input here is " << in_shape.size() + << ", but the number of dimension should be 5."; + auto out = cur->get_output_tensor(); + auto weights = cur->get_input_tensor("weights"); + auto w_shape = weights->get_shape(); + UNI_LOG_CHECK(w_shape.size() == 5, XIR_INVALID_ARG_OCCUR) + << "The number of dimension of weights here is " << w_shape.size() + << ", but the number of dimension should be 5."; + auto attrs = cur->get_attrs(); + auto kernel = attrs->get_attr>("kernel"); + auto stride = attrs->get_attr>("stride"); + std::vector dilation = {1, 1, 1}; + if (attrs->has_attr("dilation")) { + auto tmp = attrs->get_attr>("dilation"); + std::transform(tmp.begin(), tmp.end(), dilation.begin(), + [](auto t) { return t; }); + } + auto pad_mode = attrs->get_attr("pad_mode"); + std::vector padding = {0, 0, 0, 0, 0, 0}; + if (attrs->has_attr("pad")) { + auto tmp = attrs->get_attr>("pad"); + UNI_LOG_CHECK(tmp.size() == 6, XIR_INVALID_ARG_OCCUR) + << "The number of dimension of paddings here is " << tmp.size() + << ", but the number of dimension should be 6."; + std::transform(tmp.begin(), tmp.end(), padding.begin(), + [](auto i) { return i; }); + } + UNI_LOG_CHECK(w_shape[4] == in_shape[4], XIR_INVALID_ARG_OCCUR) + << "The channel of weights and the channel of input feature maps should be " + "the same." + << " But they are " << w_shape[4] << " and " << in_shape[4] << " now."; + auto oh_ow_od = cal_out_3d(in_shape, padding, kernel, stride, dilation, pad_mode); + std::vector new_out_shape = {in->get_shape().at(0), oh_ow_od[0], + oh_ow_od[1], oh_ow_od[2], w_shape[4]}; + auto output_tensor = + xir::Tensor::create(out->get_name(), new_out_shape, out->get_data_type()); + output_tensor->set_attrs(out->get_attrs()); + cur->replace_output_tensor(std::move(output_tensor)); +} + +void shape_infer_transposed_conv3d(xir::Op* cur) { + auto in = cur->get_input_tensor("input"); + auto in_shape = in->get_shape(); + UNI_LOG_CHECK(in_shape.size() == 5, XIR_INVALID_ARG_OCCUR) + << "The number of dimension of input here is " << in_shape.size() + << ", but the number of dimension should be 5."; + auto weights = cur->get_input_tensor("weights"); + auto w_shape = weights->get_shape(); + UNI_LOG_CHECK(w_shape.size() == 5, XIR_INVALID_ARG_OCCUR) + << "The number of dimension of weights here is " << w_shape.size() + << ", but the number of dimension should be 5."; + auto attrs = cur->get_attrs(); + auto kernel = attrs->get_attr>("kernel"); + auto stride = attrs->get_attr>("stride"); + std::vector dilation = {1, 1, 1}; + if (attrs->has_attr("dilation")) { + auto tmp = attrs->get_attr>("dilation"); + std::transform(tmp.begin(), tmp.end(), dilation.begin(), + [](auto t) { return t; }); + } + std::vector padding = {0, 0, 0, 0, 0, 0}; + if (attrs->has_attr("pad")) { + auto tmp = attrs->get_attr>("pad"); + UNI_LOG_CHECK(tmp.size() == 6, XIR_INVALID_ARG_OCCUR) + << "The number of dimension of paddings here is " << tmp.size() + << ", but the number of dimension should be 6."; + std::transform(tmp.begin(), tmp.end(), padding.begin(), + [](auto i) { return i; }); + } + UNI_LOG_CHECK(w_shape[4] == in_shape[4], XIR_INVALID_ARG_OCCUR) + << "The channel of weights and the channel of input feature maps should be " + "the same." + << " But they are " << w_shape[4] << " and " << in_shape[4] << " now."; + auto pad_mode = attrs->get_attr("pad_mode"); + int ow = 0, oh = 0, od = 0, oc = 0; + if (pad_mode == "FLOOR" || pad_mode == "CEIL") { + oh = (in_shape[1] - 1) * stride[1] + dilation[1] * (kernel[1] - 1) + 1 - + padding[2] - padding[3]; + ow = (in_shape[2] - 1) * stride[0] + dilation[0] * (kernel[0] - 1) + 1 - + padding[0] - padding[1]; + od = (in_shape[3] - 1) * stride[2] + dilation[2] * (kernel[2] - 1) + 1 - + padding[4] - padding[5]; + } else if (pad_mode == "SAME") { + oh = in_shape[1] * stride[1]; + ow = in_shape[2] * stride[0]; + od = in_shape[3] * stride[2]; + } else if (pad_mode == "VALID") { + oh = (in_shape[1] - 1) * stride[1] + kernel[1]; + ow = (in_shape[2] - 1) * stride[0] + kernel[0]; + od = (in_shape[3] - 1) * stride[2] + kernel[2]; + } else { + UNI_LOG_CHECK(pad_mode == "FLOOR" || pad_mode == "CEIL" || + pad_mode == "SAME" || pad_mode == "VALID", + XIR_INVALID_ARG_OCCUR) + << "pad_mode here is " << pad_mode + << ", but it should be one of \"FLOOR\"," + "\"CEIL\", \"SAME\" or \"VALID\"."; + } + oc = w_shape[0]; + auto out = cur->get_output_tensor(); + auto output_tensor = + xir::Tensor::create(out->get_name(), {in->get_shape().at(0), oh, ow, od, oc}, + out->get_data_type()); + output_tensor->set_attrs(out->get_attrs()); + cur->replace_output_tensor(std::move(output_tensor)); +} + +void shape_infer_transposed_depthwise_conv3d(xir::Op* cur) { + auto in = cur->get_input_tensor("input"); + auto in_shape = in->get_shape(); + UNI_LOG_CHECK(in_shape.size() == 5, XIR_INVALID_ARG_OCCUR) + << "The number of dimension of input here is " << in_shape.size() + << ", but the number of dimension should be 5."; + auto weights = cur->get_input_tensor("weights"); + auto w_shape = weights->get_shape(); + UNI_LOG_CHECK(w_shape.size() == 5, XIR_INVALID_ARG_OCCUR) + << "The number of dimension of weights here is " << w_shape.size() + << ", but the number of dimension should be 5."; + auto attrs = cur->get_attrs(); + auto kernel = attrs->get_attr>("kernel"); + auto stride = attrs->get_attr>("stride"); + std::vector dilation = {1, 1, 1}; + if (attrs->has_attr("dilation")) { + auto tmp = attrs->get_attr>("dilation"); + std::transform(tmp.begin(), tmp.end(), dilation.begin(), + [](auto t) { return t; }); + } + std::vector padding = {0, 0, 0, 0, 0, 0}; + if (attrs->has_attr("pad")) { + auto tmp = attrs->get_attr>("pad"); + UNI_LOG_CHECK(tmp.size() == 6, XIR_INVALID_ARG_OCCUR) + << "The number of dimension of paddings here is " << tmp.size() + << ", but the number of dimension should be 6."; + std::transform(tmp.begin(), tmp.end(), padding.begin(), + [](auto i) { return i; }); + } + UNI_LOG_CHECK(w_shape[4] == in_shape[4], XIR_INVALID_ARG_OCCUR) + << "The channel of weights and the channel of input feature maps should be " + "the same." + << " But they are " << w_shape[4] << " and " << in_shape[4] << " now."; + auto pad_mode = attrs->get_attr("pad_mode"); + int ow = 0, oh = 0, od = 0, oc = 0; + if (pad_mode == "FLOOR" || pad_mode == "CEIL") { + oh = (in_shape[1] - 1) * stride[1] + dilation[1] * (kernel[1] - 1) + 1 - + padding[2] - padding[3]; + ow = (in_shape[2] - 1) * stride[0] + dilation[0] * (kernel[0] - 1) + 1 - + padding[0] - padding[1]; + od = (in_shape[3] - 1) * stride[2] + dilation[2] * (kernel[2] - 1) + 1 - + padding[4] - padding[5]; + } else if (pad_mode == "SAME") { + oh = in_shape[1] * stride[1]; + ow = in_shape[2] * stride[0]; + od = in_shape[3] * stride[2]; + } else if (pad_mode == "VALID") { + oh = (in_shape[1] - 1) * stride[1] + kernel[1]; + ow = (in_shape[2] - 1) * stride[0] + kernel[0]; + od = (in_shape[3] - 1) * stride[2] + kernel[2]; + } else { + UNI_LOG_CHECK(pad_mode == "FLOOR" || pad_mode == "CEIL" || + pad_mode == "SAME" || pad_mode == "VALID", + XIR_INVALID_ARG_OCCUR) + << "pad_mode here is " << pad_mode + << ", but it should be one of \"FLOOR\"," + "\"CEIL\", \"SAME\" or \"VALID\"."; + } + oc = w_shape[4]; + auto out = cur->get_output_tensor(); + auto output_tensor = + xir::Tensor::create(out->get_name(), {in->get_shape().at(0), oh, ow, od, oc}, + out->get_data_type()); + output_tensor->set_attrs(out->get_attrs()); + cur->replace_output_tensor(std::move(output_tensor)); +} + void shape_infer_pool2d(xir::Op* cur) { auto in = cur->get_input_tensor("input"); auto in_shape = in->get_shape(); @@ -436,7 +720,7 @@ void shape_infer_concat(xir::Op* cur) { auto axis = cur->get_attr("axis"); auto ins = cur->get_input_tensors(); axis = axis < 0 ? axis + ins[0]->get_shape().size() : axis; - for (uint i = 0; i < ins.size(); i++) + for (unsigned int i = 0; i < ins.size(); i++) for (auto d = 0; d < static_cast(ins[0]->get_shape().size()); d++) if (d != axis) @@ -592,7 +876,7 @@ void shape_infer_reshape(xir::Op* cur) { int size_r = 1; auto in = cur->get_input_tensor("input"); - for (uint i = 0; i < reshape.size(); i++) { + for (unsigned int i = 0; i < reshape.size(); i++) { auto dim = reshape[i]; if (dim <= 0) { size_r *= 1; @@ -604,7 +888,7 @@ void shape_infer_reshape(xir::Op* cur) { } int value = in->get_element_num() / size_r; - for (uint i = 0; i < reshape.size(); i++) { + for (unsigned int i = 0; i < reshape.size(); i++) { if (reshape[i] < 0) reshape[i] = value; else if (reshape[i] == 0) @@ -652,7 +936,7 @@ void shape_infer_pad(xir::Op* cur) { XIR_UNEXPECTED_VALUE); auto out = cur->get_output_tensor(); std::vector out_shape; - for (uint i = 0; i < in->get_shape().size(); i++) { + for (unsigned int i = 0; i < in->get_shape().size(); i++) { auto shape = in->get_shape().at(i) + padding[2 * i] + padding[2 * i + 1]; out_shape.push_back(shape); } @@ -666,7 +950,7 @@ void shape_infer_reduction(xir::Op* cur) auto in = cur->get_input_tensor("input"); auto dims = cur->get_attr>("axis"); auto keep_dims = cur->get_attr("keep_dims"); - std::vector bitmap = {false, false, false, false}; + std::vector bitmap(dims.size(), false); std::int32_t dim_size = in->get_shape().size(); for (auto index : dims) { UNI_LOG_CHECK(index >= -dim_size && index < dim_size, XIR_INVALID_ARG_OCCUR) @@ -763,52 +1047,111 @@ void shape_infer_resize(xir::Op* cur) { auto in = cur->get_input_tensor("input"); auto out = cur->get_output_tensor(); std::vector out_shape = in->get_shape(); - + auto mode = cur->get_attr("mode"); if (cur->get_input_num() > 1) { auto ops = internal::vec_input_ops(cur->get_input_ops()); for (auto op : ops) { auto out_tensor = op->get_output_tensor(); if (op->has_attr("data")) { auto size = read_data_in_attr(op); - UNI_LOG_CHECK(size.size() == 2 || size.size() == 4, - XIR_INVALID_ARG_OCCUR) - << "i don't know how to resize the feature maps"; - if (size.size() == 4) - out_shape = size; - else if (size.size() == 2) { - out_shape[1] = size[0]; - out_shape[2] = size[1]; + if (size.size() == 2 || size.size() == 4) { + UNI_LOG_CHECK(mode == "NEAREST" || mode == "BILINEAR", + XIR_INVALID_ARG_OCCUR) + << "the dimension number of the output feature maps of resize " + "operation is " + << size.size() << ", the mode of resize is " << mode + << ". We support dimension number of output feature maps to be 2 " + "or 4 for NEAREST and BILINEAR, 3 or 5 for TRILINEAR."; + if (size.size() == 2) { + out_shape[1] = size[0]; + out_shape[2] = size[1]; + } else { + UNI_LOG_CHECK(out_shape[0] == size[0] && out_shape[3] == size[3], + XIR_INVALID_ARG_OCCUR) + << "resize 4-d feature maps could only implement along the " + "diemnsion of the width and the height."; + out_shape = size; + } + } else if (size.size() == 3 || size.size() == 5) { + UNI_LOG_CHECK(mode == "TRILINEAR", XIR_INVALID_ARG_OCCUR) + << "the dimension number of the output feature maps of resize " + "operation is " + << size.size() << ", the mode of resize is " << mode + << ". We support dimension number of output feature maps to be 2 " + "or 4 for NEAREST and BILINEAR, 3 or 5 for TRILINEAR."; + if (size.size() == 3) { + out_shape[1] = size[0]; + out_shape[2] = size[1]; + out_shape[3] = size[2]; + } else { + UNI_LOG_CHECK(out_shape[0] == size[0] && out_shape[4] == size[4], + XIR_INVALID_ARG_OCCUR) + << "resize 5-d feature maps could only implement along the " + "diemnsion of the width, the height and the depth."; + out_shape = size; + } } - UNI_LOG_CHECK( - out_shape.size() == static_cast(in->get_shape().size()), - XIR_INVALID_ARG_OCCUR) + UNI_LOG_CHECK(out_shape.size() == + static_cast(in->get_shape().size()), + XIR_INVALID_ARG_OCCUR) << "the number of dimensions shoud be the same after the resize " "op."; } else if (out_tensor->has_attr("shape_info")) { auto size = - out_tensor->get_attr>("shape_info"); - UNI_LOG_CHECK(size.size() == 2, XIR_INVALID_ARG_OCCUR) - << "New size of feature maps should have two dimensions H, W."; + out_tensor->get_attr>("shape_info"); + UNI_LOG_CHECK(size.size() == 2 || size.size() == 3, + XIR_INVALID_ARG_OCCUR) + << "New size of feature maps should have two dimensions H, W or " + "three dimensions H, W, D."; out_shape[1] = size[0]; out_shape[2] = size[1]; + if (size.size() == 3) out_shape[3] = size[2]; } + UNI_LOG_CHECK( + out_shape.size() == static_cast(in->get_shape().size()), + XIR_INVALID_ARG_OCCUR) + << "the number of dimensions shoud be the same after the resize " + "op."; } } else if (cur->has_attr("scale")) { - auto scale = cur->get_attr>("scale"); - UNI_LOG_CHECK(scale.size() == 2, XIR_INVALID_ARG_OCCUR) - << "Scale should have two dimensions (scale_w, scale_h)."; - UNI_LOG_CHECK(scale[0] > 0 && scale[1] > 0, XIR_INVALID_ARG_OCCUR) - << "Scale should be > 0."; + auto scale = get_float_vec_from_any(cur->get_attr("scale")); + UNI_LOG_CHECK(scale.size() == 2 || scale.size() == 3, XIR_INVALID_ARG_OCCUR) + << "Scale should have two dimensions (scale_w, scale_h) or three " + "dimensions (scale_w, scale_h, scale_d)."; + UNI_LOG_CHECK( + std::all_of(scale.begin(), scale.end(), [](auto s) { return s > 0; }), + XIR_INVALID_ARG_OCCUR) + << "Scale should be > 0."; + if (scale.size() == 2) { + UNI_LOG_CHECK(mode == "NEAREST" || mode == "BILINEAR", + XIR_INVALID_ARG_OCCUR) + << "the number of the attribute scale of resize " + "operation is " + << scale.size() << ", the mode of resize is " << mode + << ". We support the number of scale to be 2 for NEAREST and " + "BILINEAR, 3 for TRILINEAR."; + } else if (scale.size() == 3) { + UNI_LOG_CHECK(mode == "TRILINEAR", XIR_INVALID_ARG_OCCUR) + << "the number of the attribute scale of resize " + "operation is " + << scale.size() << ", the mode of resize is " << mode + << ". We support the number of scale to be 2 for NEAREST and " + "BILINEAR, 3 for TRILINEAR."; + } auto ow = static_cast(in->get_shape().at(2) * scale[0]); auto oh = static_cast(in->get_shape().at(1) * scale[1]); out_shape[1] = oh; out_shape[2] = ow; + if (scale.size() == 3) { + auto od = static_cast(in->get_shape().at(3) * scale[2]); + out_shape[3] = od; + } } else { UNI_LOG_ERROR(XIR_INVALID_ARG_OCCUR) - << "I don't know how to resize the feature maps."; + << "I don't know how to resize the feature maps."; } auto output_tensor = - xir::Tensor::create(out->get_name(), out_shape, out->get_data_type()); + xir::Tensor::create(out->get_name(), out_shape, out->get_data_type()); output_tensor->set_attrs(out->get_attrs()); cur->replace_output_tensor(std::move(output_tensor)); } @@ -848,6 +1191,45 @@ void shape_infer_gstiling(xir::Op* cur) cur->replace_output_tensor(std::move(output_tensor)); } +void shape_infer_pixel_shuffle(xir::Op* cur) { + auto upscale = cur->get_attr("upscale"); + auto scale = cur->get_attr("scale"); + auto in_shape = cur->get_input_tensor("input")->get_shape(); + auto ic = in_shape[3]; + auto iw = in_shape[2]; + auto ih = in_shape[1]; + auto scale_sq = scale * scale; + std::int32_t ow, oh, oc; + if (upscale) { + oc = ic / scale_sq; + ow = iw * scale; + oh = ih * scale; + UNI_LOG_CHECK(ic % scale_sq == 0, XIR_INVALID_ARG_OCCUR) + << "The number of input channels for pixel_shuffle layer must be multiples " + << "of the scale * scale."; + } else { + oc = ic * scale_sq; + ow = iw / scale; + oh = ih / scale; + UNI_LOG_CHECK(iw % scale == 0, XIR_INVALID_ARG_OCCUR) + << "The number of input width for pixel_shuffle layer must be multiples " + << "of the scale."; + UNI_LOG_CHECK(ih % scale == 0, XIR_INVALID_ARG_OCCUR) + << "The number of input height for pixel_shuffle layer must be multiples " + << "of the scale."; + } + std::vector out_shape = {in_shape[0], oh, ow, oc}; + auto out = cur->get_output_tensor(); + auto output_tensor = + xir::Tensor::create(out->get_name(), out_shape, out->get_data_type()); + output_tensor->set_attrs(out->get_attrs()); + cur->replace_output_tensor(std::move(output_tensor)); +} + +void shape_infer_pixel_shuffle_fix(xir::Op* cur) { + shape_infer_pixel_shuffle(cur); +} + void shape_infer_batchnorm(xir::Op* cur) { auto in_shape = cur->get_input_tensor("input")->get_shape(); auto gamma = cur->get_input_tensor("gamma")->get_shape(); @@ -918,7 +1300,7 @@ void shape_infer_broadcast(xir::Op* cur) << xir::to_string(input_tensors[0]->get_shape()) << " and another is " << xir::to_string(input_tensors[1]->get_shape()) << "."; if (in_num > 2) { - for (uint i = 2; i < input_tensors.size(); i++) { + for (unsigned int i = 2; i < input_tensors.size(); i++) { auto out = size_broadcast(std::get<1>(sbc_rlt), input_tensors[i]->get_shape()); UNI_LOG_CHECK(std::get<0>(out), XIR_INVALID_ARG_OCCUR) @@ -952,7 +1334,8 @@ void forward_mul(xir::Op* cur) { } std::vector value; if (mul0.size() == mul1.size()) - for (uint i = 0; i < mul0.size(); ++i) value.push_back(mul0[i] * mul1[i]); + for (unsigned int i = 0; i < mul0.size(); ++i) + value.push_back(mul0[i] * mul1[i]); auto out = cur->get_output_tensor(); out->set_attr("shape_info", value); } @@ -972,10 +1355,12 @@ void forward_div(xir::Op* cur) { auto shape = inv->get_attr>("shape_info"); auto base = read_data_in_attr(cur->get_input_ops("input")[1]); if (base.size() == 1) - for (uint i = 0; i < shape.size(); i++) shape[i] = shape[i] / base[0]; + for (unsigned int i = 0; i < shape.size(); i++) + shape[i] = shape[i] / base[0]; else { UNI_LOG_CHECK(shape.size() == base.size(), XIR_INVALID_ARG_OCCUR); - for (uint i = 0; i < shape.size(); i++) shape[i] = shape[i] / base[i]; + for (unsigned int i = 0; i < shape.size(); i++) + shape[i] = shape[i] / base[i]; } auto out = cur->get_output_tensor(); out->set_attr("shape_info", shape); @@ -1017,7 +1402,8 @@ void forward_strided_slice(xir::Op* cur, } } if (indata == nullptr) return; - UNI_LOG_CHECK(begin.size() == uint(in_shape_num), XIR_INVALID_ARG_OCCUR) + UNI_LOG_CHECK(begin.size() == (unsigned int)in_shape_num, + XIR_INVALID_ARG_OCCUR) << "the size of begin is: " << begin.size() << ", the size of input tensor is: " << in_shape_num << "."; std::vector idx_stride{1}; @@ -1453,7 +1839,7 @@ void shape_infer_flatten(xir::Op* cur) { std::vector new_size; for (auto i = 0; i < start_dim; i++) new_size.push_back(input_size[i]); new_size.push_back(flattened_dim); - for (uint i = end_dim + 1; i < input_size.size(); i++) + for (unsigned int i = end_dim + 1; i < input_size.size(); i++) new_size.push_back(input_size[i]); auto new_tensor = xir::Tensor::create(tensor->get_name(), new_size, tensor->get_data_type()); @@ -1475,7 +1861,7 @@ void shape_infer_transpose(xir::Op* cur) auto output_tensor = cur->get_output_tensor(); auto order = cur->get_attr>("order"); std::vector output_shape; - for (uint i = 0; i < order.size(); i++) + for (unsigned int i = 0; i < order.size(); i++) output_shape.push_back(input_shape[order[i]]); auto new_tensor = xir::Tensor::create(output_tensor->get_name(), output_shape, output_tensor->get_data_type()); @@ -1499,10 +1885,10 @@ void shape_infer_priorbox(xir::Op* cur) std::vector aspect_ratios_; aspect_ratios_.push_back(1.); auto flip = cur->get_attr("flip"); - for (uint i = 0; i < aspect_ratios.size(); ++i) { + for (unsigned int i = 0; i < aspect_ratios.size(); ++i) { float ar = aspect_ratios[i]; bool already_exist = false; - for (uint j = 0; j < aspect_ratios_.size(); ++j) { + for (unsigned int j = 0; j < aspect_ratios_.size(); ++j) { if (fabs(ar - aspect_ratios_[j]) < 1e-6) { already_exist = true; break; @@ -1517,7 +1903,7 @@ void shape_infer_priorbox(xir::Op* cur) } int num_priors_ = aspect_ratios_.size() * min_sizes.size(); if (max_sizes.size() > 0) { - for (uint i = 0; i < max_sizes.size(); ++i) { + for (unsigned int i = 0; i < max_sizes.size(); ++i) { num_priors_ += 1; } } @@ -1627,13 +2013,15 @@ void shape_infer_conv2d_fix(xir::Op* cur std::transform(tmp.begin(), tmp.end(), padding.begin(), [](auto i) { return i; }); } - oh = std::ceil((in_shape[1] + padding[2] + padding[3] - - (kernel[1] - 1) * dilation[1] - 1) / - stride[1]) + + oh = std::floor(1.0f * + (in_shape[1] + padding[2] + padding[3] - + (kernel[1] - 1) * dilation[1] - 1) / + stride[1]) + 1; - ow = std::ceil((in_shape[2] + padding[0] + padding[1] - - (kernel[0] - 1) * dilation[0] - 1) / - stride[0]) + + ow = std::floor(1.0f * + (in_shape[2] + padding[0] + padding[1] - + (kernel[0] - 1) * dilation[0] - 1) / + stride[0]) + 1; oc = w_shape[0]; std::vector new_out_shape = {in->get_shape().at(0), oh, ow, oc}; @@ -1672,13 +2060,15 @@ void shape_infer_depthwise_conv2d_fix(xi std::transform(tmp.begin(), tmp.end(), padding.begin(), [](auto i) { return i; }); } - oh = std::ceil((in_shape[1] + padding[2] + padding[3] - - (kernel[1] - 1) * dilation[1] - 1) / - stride[1]) + + oh = std::floor(1.0f * + (in_shape[1] + padding[2] + padding[3] - + (kernel[1] - 1) * dilation[1] - 1) / + stride[1]) + 1; - ow = std::ceil((in_shape[2] + padding[0] + padding[1] - - (kernel[0] - 1) * dilation[0] - 1) / - stride[0]) + + ow = std::floor(1.0f * + (in_shape[2] + padding[0] + padding[1] - + (kernel[0] - 1) * dilation[0] - 1) / + stride[0]) + 1; oc = w_shape[0] * w_shape[3]; std::vector new_out_shape = {in->get_shape().at(0), oh, ow, oc}; @@ -1766,6 +2156,192 @@ void shape_infer_transposed_depthwise_co cur->replace_output_tensor(std::move(output_tensor)); } +void shape_infer_conv3d_fix(xir::Op* cur) { + auto in = cur->get_input_tensor("input"); + auto in_shape = in->get_shape(); + auto out = cur->get_output_tensor(); + auto weights = cur->get_input_tensor("weights"); + auto w_shape = weights->get_shape(); + UNI_LOG_CHECK(w_shape.size() == 5, XIR_INVALID_ARG_OCCUR) + << "Op" << cur->to_string() << ". The size of dimension of weights here is " + << w_shape.size() << ", but the size of dimension should be 5."; + auto attrs = cur->get_attrs(); + auto kernel = attrs->get_attr>("kernel"); + auto stride = attrs->get_attr>("stride"); + std::vector dilation = {1, 1, 1}; + if (attrs->has_attr("dilation")) { + auto tmp = attrs->get_attr>("dilation"); + std::transform( + tmp.begin(), tmp.end(), dilation.begin(), [](auto t) { return t; }); + } + int ow = 0, oh = 0, od = 0, oc = 0; + std::vector padding = {0, 0, 0, 0, 0, 0}; + if (attrs->has_attr("pad")) { + auto tmp = attrs->get_attr>("pad"); + UNI_LOG_CHECK(tmp.size() == 6, XIR_INVALID_ARG_OCCUR) + << "The size of dimension of paddings here is " << tmp.size() + << ", but the size of dimension should be 6."; + std::transform( + tmp.begin(), tmp.end(), padding.begin(), [](auto i) { return i; }); + } + oh = std::floor(1.0 * + (in_shape[1] + padding[2] + padding[3] - + (kernel[1] - 1) * dilation[1] - 1) / + stride[1]) + + 1; + ow = std::floor(1.0 * + (in_shape[2] + padding[0] + padding[1] - + (kernel[0] - 1) * dilation[0] - 1) / + stride[0]) + + 1; + od = std::floor(1.0 * + (in_shape[3] + padding[4] + padding[5] - + (kernel[2] - 1) * dilation[2] - 1) / + stride[2]) + + 1; + oc = w_shape[0]; + std::vector new_out_shape = { + in->get_shape().at(0), oh, ow, od, oc}; + auto output_tensor = + xir::Tensor::create(out->get_name(), new_out_shape, out->get_data_type()); + output_tensor->set_attrs(out->get_attrs()); + cur->replace_output_tensor(std::move(output_tensor)); +} + +void shape_infer_depthwise_conv3d_fix(xir::Op* cur) { + auto in = cur->get_input_tensor("input"); + auto in_shape = in->get_shape(); + auto out = cur->get_output_tensor(); + auto weights = cur->get_input_tensor("weights"); + auto w_shape = weights->get_shape(); + UNI_LOG_CHECK(w_shape.size() == 5, XIR_INVALID_ARG_OCCUR) + << "Op" << cur->to_string() << ". The size of dimension of weights here is " + << w_shape.size() << ", but the size of dimension should be 5."; + auto attrs = cur->get_attrs(); + auto kernel = attrs->get_attr>("kernel"); + auto stride = attrs->get_attr>("stride"); + std::vector dilation = {1, 1, 1}; + if (attrs->has_attr("dilation")) { + auto tmp = attrs->get_attr>("dilation"); + std::transform( + tmp.begin(), tmp.end(), dilation.begin(), [](auto t) { return t; }); + } + int ow = 0, oh = 0, od = 0, oc = 0; + std::vector padding = {0, 0, 0, 0, 0, 0}; + if (attrs->has_attr("pad")) { + auto tmp = attrs->get_attr>("pad"); + UNI_LOG_CHECK(tmp.size() == 6, XIR_INVALID_ARG_OCCUR) + << "The size of dimension of paddings here is " << tmp.size() + << ", but the size of dimension should be 6."; + std::transform( + tmp.begin(), tmp.end(), padding.begin(), [](auto i) { return i; }); + } + oh = std::floor(1.0 * + (in_shape[1] + padding[2] + padding[3] - + (kernel[1] - 1) * dilation[1] - 1) / + stride[1]) + + 1; + ow = std::floor(1.0 * + (in_shape[2] + padding[0] + padding[1] - + (kernel[0] - 1) * dilation[0] - 1) / + stride[0]) + + 1; + od = std::floor(1.0 * + (in_shape[3] + padding[4] + padding[5] - + (kernel[2] - 1) * dilation[2] - 1) / + stride[2]) + + 1; + oc = w_shape[4]; + std::vector new_out_shape = { + in->get_shape().at(0), oh, ow, od, oc}; + auto output_tensor = + xir::Tensor::create(out->get_name(), new_out_shape, out->get_data_type()); + output_tensor->set_attrs(out->get_attrs()); + cur->replace_output_tensor(std::move(output_tensor)); +} + +void shape_infer_transposed_conv3d_fix(xir::Op* cur) { + auto in = cur->get_input_tensor("input"); + auto in_shape = in->get_shape(); + auto weights = cur->get_input_tensor("weights"); + auto w_shape = weights->get_shape(); + auto attrs = cur->get_attrs(); + auto kernel = attrs->get_attr>("kernel"); + auto stride = attrs->get_attr>("stride"); + std::vector dilation = {1, 1, 1}; + if (attrs->has_attr("dilation")) { + auto tmp = attrs->get_attr>("dilation"); + std::transform(tmp.begin(), tmp.end(), dilation.begin(), + [](auto t) { return t; }); + } + std::vector padding = {0, 0, 0, 0, 0, 0}; + if (attrs->has_attr("pad")) { + auto tmp = attrs->get_attr>("pad"); + UNI_LOG_CHECK(tmp.size() == 6, XIR_INVALID_ARG_OCCUR) + << "The number of dimension of paddings here is " << tmp.size() + << ", but the number of dimension should be 6."; + std::transform(tmp.begin(), tmp.end(), padding.begin(), + [](auto i) { return i; }); + } + std::int32_t ow, oh, od, oc; + // copied from + // https://github.com/BVLC/caffe/blob/master/src/caffe/layers/deconv_layer.cpp + ow = (in_shape[2] - 1) * stride[0] + dilation[0] * (kernel[0] - 1) + 1 - + padding[0] - padding[1]; + oh = (in_shape[1] - 1) * stride[1] + dilation[1] * (kernel[1] - 1) + 1 - + padding[2] - padding[3]; + od = (in_shape[3] - 1) * stride[2] + dilation[2] * (kernel[2] - 1) + 1 - + padding[4] - padding[5]; + oc = w_shape[0]; + auto out = cur->get_output_tensor(); + auto output_tensor = + xir::Tensor::create(out->get_name(), {in->get_shape().at(0), oh, ow, od, oc}, + out->get_data_type()); + output_tensor->set_attrs(out->get_attrs()); + cur->replace_output_tensor(std::move(output_tensor)); +} + +void shape_infer_transposed_depthwise_conv3d_fix(xir::Op* cur) { + auto in = cur->get_input_tensor("input"); + auto in_shape = in->get_shape(); + auto weights = cur->get_input_tensor("weights"); + auto w_shape = weights->get_shape(); + auto attrs = cur->get_attrs(); + auto kernel = attrs->get_attr>("kernel"); + auto stride = attrs->get_attr>("stride"); + std::vector dilation = {1, 1, 1}; + if (attrs->has_attr("dilation")) { + auto tmp = attrs->get_attr>("dilation"); + std::transform(tmp.begin(), tmp.end(), dilation.begin(), + [](auto t) { return t; }); + } + std::vector padding = {0, 0, 0, 0, 0, 0}; + if (attrs->has_attr("pad")) { + auto tmp = attrs->get_attr>("pad"); + UNI_LOG_CHECK(tmp.size() == 6, XIR_INVALID_ARG_OCCUR) + << "The number of dimension of paddings here is " << tmp.size() + << ", but the number of dimension should be 6."; + std::transform(tmp.begin(), tmp.end(), padding.begin(), + [](auto i) { return i; }); + } + std::int32_t ow, oh, od, oc; + // copied from + // https://github.com/BVLC/caffe/blob/master/src/caffe/layers/deconv_layer.cpp + ow = (in_shape[2] - 1) * stride[0] + dilation[0] * (kernel[0] - 1) + 1 - + padding[0] - padding[1]; + oh = (in_shape[1] - 1) * stride[1] + dilation[1] * (kernel[1] - 1) + 1 - + padding[2] - padding[3]; + od = (in_shape[3] - 1) * stride[2] + dilation[2] * (kernel[2] - 1) + 1 - + padding[4] - padding[5]; + oc = w_shape[4]; + auto out = cur->get_output_tensor(); + auto output_tensor = + xir::Tensor::create(out->get_name(), {in->get_shape().at(0), oh, ow, od, oc}, + out->get_data_type()); + output_tensor->set_attrs(out->get_attrs()); + cur->replace_output_tensor(std::move(output_tensor)); +} + void shape_infer_const_fix(xir::Op* cur) { shape_infer_const(cur); } void shape_infer_data_fix(xir::Op* cur) { shape_infer_data(cur); } @@ -1799,11 +2375,11 @@ void shape_infer_pool_fix(xir::Op* cur) std::transform(tmp.begin(), tmp.end(), padding.begin(), [](auto i) { return i; }); } - oh = std::ceil((in_shape[1] + padding[2] + padding[3] - kernel[1]) / - stride[1]) + + oh = std::floor(1.0f * (in_shape[1] + padding[2] + padding[3] - kernel[1]) / + stride[1]) + 1; - ow = std::ceil((in_shape[2] + padding[0] + padding[1] - kernel[0]) / - stride[0]) + + ow = std::floor(1.0f * (in_shape[2] + padding[0] + padding[1] - kernel[0]) / + stride[0]) + 1; oc = in_shape[3]; std::vector new_out_shape = {in->get_shape().at(0), oh, ow, oc}; @@ -1819,7 +2395,7 @@ static void shape_infer_updownsample_fix auto op_type = cur->get_type(); UNI_LOG_CHECK(("downsample-fix" == op_type) || ("upsample-fix" == op_type), XIR_UNEXPECTED_VALUE); - auto scale = cur->get_attr>("scale"); + auto scale = get_float_vec_from_any(cur->get_attr("scale")); auto scale_w = scale.at(0); auto scale_h = scale.at(1); auto input_shape = cur->get_input_tensors("input").at(0)->get_shape(); diff -pruN 1.4-2/src/xir/op/shape_inference.hpp 2.5-1/src/xir/op/shape_inference.hpp --- 1.4-2/src/xir/op/shape_inference.hpp 2021-07-27 05:41:52.000000000 +0000 +++ 2.5-1/src/xir/op/shape_inference.hpp 2022-06-15 07:33:00.000000000 +0000 @@ -16,6 +16,7 @@ #pragma once +#include #include #include #include @@ -29,6 +30,10 @@ void shape_infer_conv2d(xir::Op* cur); void shape_infer_depthwise_conv2d(xir::Op* cur); void shape_infer_transposed_conv2d(xir::Op* cur); void shape_infer_transposed_depthwise_conv2d(xir::Op* cur); +void shape_infer_conv3d(xir::Op* cur); +void shape_infer_depthwise_conv3d(xir::Op* cur); +void shape_infer_transposed_conv3d(xir::Op* cur); +void shape_infer_transposed_depthwise_conv3d(xir::Op* cur); void shape_infer_pool2d(xir::Op* cur); void shape_infer_maxpool2d(xir::Op* cur); void shape_infer_avgpool2d(xir::Op* cur); @@ -82,6 +87,7 @@ void shape_infer_priorbox(xir::Op* cur); void shape_infer_stack(xir::Op* cur); void shape_infer_matmul(xir::Op* cur); void shape_infer_gstiling(xir::Op* cur); +void shape_infer_pixel_shuffle(xir::Op* cur); void shape_infer_exp(xir::Op* cur); void shape_infer_neg(xir::Op* cur); void shape_infer_scale(xir::Op* cur); @@ -96,8 +102,12 @@ void shape_infer_ddr_flatten_concat(xir: // shape_infer function for fixed ops void shape_infer_conv2d_fix(xir::Op* cur); void shape_infer_depthwise_conv2d_fix(xir::Op* cur); +void shape_infer_depthwise_conv3d_fix(xir::Op* cur); void shape_infer_transposed_conv2d_fix(xir::Op* cur); void shape_infer_transposed_depthwise_conv2d_fix(xir::Op* cur); +void shape_infer_conv3d_fix(xir::Op* cur); +void shape_infer_transposed_conv3d_fix(xir::Op* cur); +void shape_infer_transposed_depthwise_conv3d_fix(xir::Op* cur); void shape_infer_const_fix(xir::Op* cur); void shape_infer_data_fix(xir::Op* cur); void shape_infer_split_fix(xir::Op* cur); @@ -109,6 +119,7 @@ void shape_infer_reorg_fix(xir::Op* cur) void shape_infer_ddr_flatten_concat_fix(xir::Op* cur); void shape_infer_reshape_fix(xir::Op* cur); void shape_infer_tile_fix(xir::Op* cur); +void shape_infer_pixel_shuffle_fix(xir::Op* cur); void shape_infer_pad_fix(xir::Op* cur); void shape_infer_upsample_fix(xir::Op* cur); void shape_infer_downsample_fix(xir::Op* cur); diff -pruN 1.4-2/src/xir/tensor/tensor_imp.cpp 2.5-1/src/xir/tensor/tensor_imp.cpp --- 1.4-2/src/xir/tensor/tensor_imp.cpp 2021-07-27 05:41:52.000000000 +0000 +++ 2.5-1/src/xir/tensor/tensor_imp.cpp 2022-06-15 07:33:00.000000000 +0000 @@ -134,40 +134,6 @@ Tensor* TensorImp::set_attr(const std::s return this; } -} // namespace xir - -/* C API implementations */ -#include "xir/xir.h" - -extern "C" xir_tensor_t xir_tensor_create(const char* name, const int32_t* dims, - const int32_t dim_num, - enum xir_tensor_data_type_t data_type, - const int32_t bit_width) { - std::vector v_dims(dims, dims + dim_num); - auto t = xir::Tensor::create( - std::string(name), v_dims, - xir::DataType{static_cast(data_type), bit_width}); - return static_cast(t.release()); -} -extern "C" int xir_tensor_destroy(xir_tensor_t tensor) { - auto t = static_cast(tensor); - delete t; - return 0; -} - -namespace xir { -class c_api { - public: - static const char* tensor_get_name(xir_tensor_t tensor) { - auto self = static_cast(tensor); - return self->name_.c_str(); - } - static const xir_attrs_t tensor_get_attrs(xir_tensor_t tensor) { - auto self = static_cast(tensor); - return static_cast(self->attrs_.get()); - } -}; - Tensor* TensorImp::rename(const std::string& name) { if (this->name_ != name) { if (nullptr != this->producer_) { @@ -201,33 +167,3 @@ const std::string TensorImp::to_string(c } }; // namespace xir - -extern "C" const char* xir_tensor_get_name(xir_tensor_t tensor) { - return xir::c_api::tensor_get_name(tensor); -} -extern "C" int32_t xir_tensor_get_bit_width(xir_tensor_t tensor) { - return static_cast(tensor)->get_data_type().bit_width; -} -extern "C" int32_t xir_tensor_get_dim_size(xir_tensor_t tensor, int32_t idx) { - return static_cast(tensor)->get_shape().at(idx); -} - -extern "C" int32_t xir_tensor_get_dim_num(xir_tensor_t tensor) { - return static_cast(tensor)->get_shape().size(); -} - -extern "C" enum xir_tensor_data_type_t xir_tensor_get_data_type( - xir_tensor_t tensor) { - return static_cast( - static_cast(tensor)->get_data_type().type); -} -extern "C" int32_t xir_tensor_get_element_num(xir_tensor_t tensor) { - return static_cast(tensor)->get_element_num(); -} - -extern "C" int32_t xir_tensor_get_data_size(xir_tensor_t tensor) { - return static_cast(tensor)->get_data_size(); -} -extern "C" xir_attrs_t xir_tensor_get_attrs(xir_tensor_t tensor) { - return xir::c_api::tensor_get_attrs(tensor); -} diff -pruN 1.4-2/src/xir/util/tool_function.cpp 2.5-1/src/xir/util/tool_function.cpp --- 1.4-2/src/xir/util/tool_function.cpp 2021-07-27 05:41:52.000000000 +0000 +++ 2.5-1/src/xir/util/tool_function.cpp 2022-06-15 07:33:00.000000000 +0000 @@ -15,20 +15,30 @@ */ #include "xir/util/tool_function.hpp" -#include + #include #include #include #include #include #include +#include + +#include "../../../3rd-party/hash-library/md5.h" #include "UniLog/UniLog.hpp" #include "config.hpp" #include "internal_util.hpp" +#include "xir/op/op_def.hpp" +#include "xir/op/op_def_factory_imp.hpp" +#include "xir/op/shape_inference.hpp" #include "xir/util/data_type.hpp" namespace xir { +const std::string get_md5_of_buffer(const void* buf, size_t size) { + return MD5()(buf, size); +} + const std::string get_md5_of_file(const std::string& filepath) { std::ifstream file(filepath.c_str(), std::ifstream::in); UNI_LOG_CHECK(!file.fail(), XIR_FILE_NOT_EXIST) @@ -39,24 +49,15 @@ const std::string get_md5_of_file(const const std::uint32_t buffer_size = 1024; char* buffer; buffer = new char[buffer_size]; - auto read_size = file.readsome(buffer, buffer_size); - unsigned char md5sum[MD5_DIGEST_LENGTH]; - MD5_CTX md5_context; - MD5_Init(&md5_context); + auto strm = MD5(); while (read_size) { - MD5_Update(&md5_context, buffer, read_size); + strm.add(buffer, read_size); read_size = file.readsome(buffer, buffer_size); } - MD5_Final(md5sum, &md5_context); - std::stringstream ss; - for (std::uint32_t idx = 0; idx < MD5_DIGEST_LENGTH; idx++) { - ss << std::setfill('0') << std::setw(2) << std::hex - << static_cast(md5sum[idx]); - } - const std::string ret = ss.str(); delete[] buffer; file.close(); + auto ret = strm.getHash(); UNI_LOG_DEBUG_INFO << "md5sum(" << filepath << ") = " << ret << "."; return ret; } @@ -165,4 +166,52 @@ float xround(const float& data, const st return ret; } +void register_customized_operator_definition(const std::string& name, + const std::string& type) { + UNI_LOG_WARNING + << "The operator named " << name << ", type: " << type + << ", is not defined in XIR. XIR creates the definition of this " + "operator automatically. " + << "You should specify the shape and " + "the data_type of the output tensor of this operation by " + "set_attr(\"shape\", std::vector) and " + "set_attr(\"data_type\", std::string)"; + auto new_operator = + xir::OpDef(type) + .add_input_arg(xir::OpArgDef{"input", OpArgDef::REPEATED, + xir::DataType::FLOAT, ""}) + .add_attr(xir::AttrDefBuilder>::build( + "shape", AttrDef::REQUIRED, 0, + "`Datatype`: `vector`\n\n" + "The shape of the output tensor")) + .add_attr(xir::AttrDefBuilder::build( + "data_type", AttrDef::REQUIRED, + "`Datatype`: `string`\n\n" + "The data type of the data of output feature maps, " + "we use FLOAT32 as the default.")) + .set_annotation("This operator is not defined by XIR.") + .set_shape_infer(xir::shape_infer_data); + op_def_factory()->register_h(new_operator); +} + +std::vector get_float_vec_from_any(const xir::any& any) { + auto type = std::type_index(any.type()); + auto f_vec = std::type_index(typeid(std::vector)); + auto i_vec = std::type_index(typeid(std::vector)); + std::vector fs; + if (type == i_vec) { + auto is = std::any_cast>(any); + for (auto i : is) fs.push_back(static_cast(i)); + } else if (type == f_vec) { + fs = std::any_cast>(any); + } else { + UNI_LOG_ERROR(XIR_INVALID_ARG_OCCUR) + << "I cannot transform this xir::any to float."; + } + return fs; +} + +bool TensorLexicographicalOrder(Tensor* a, Tensor* b) { + return a->get_name() < b->get_name(); +} } // namespace xir diff -pruN 1.4-2/test/CMakeLists.txt 2.5-1/test/CMakeLists.txt --- 1.4-2/test/CMakeLists.txt 2021-07-27 05:41:52.000000000 +0000 +++ 2.5-1/test/CMakeLists.txt 2022-06-15 07:33:00.000000000 +0000 @@ -1,22 +1,21 @@ # # Copyright 2019 Xilinx Inc. # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. # -include_directories( - ${CMAKE_CURRENT_BINARY_DIR}/../src/xir - ${CMAKE_SOURCE_DIR}/src) +include_directories(${CMAKE_CURRENT_BINARY_DIR}/../src/xir + ${PROJECT_SOURCE_DIR}/src) aux_source_directory(. SRC_FILS) @@ -25,8 +24,6 @@ foreach(FIL ${SRC_FILS}) set(BIN ${FIL_NAME}.bin) add_executable(${BIN} ${FIL}) - target_link_libraries(${BIN} protobuf::libprotobuf unilog::unilog glog::glog crypto ${PROJECT_NAME}) + target_link_libraries(${BIN} protobuf::libprotobuf unilog::unilog + ${PROJECT_NAME}) endforeach() - -add_executable (test_c_api test_c_api.c) -target_link_libraries (test_c_api ${PROJECT_NAME}) diff -pruN 1.4-2/test/test_c_api.c 2.5-1/test/test_c_api.c --- 1.4-2/test/test_c_api.c 2021-07-27 05:41:52.000000000 +0000 +++ 2.5-1/test/test_c_api.c 1970-01-01 00:00:00.000000000 +0000 @@ -1,185 +0,0 @@ -/* - * Copyright 2019 Xilinx Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include - -int show_name(int argc, char* argv[]) { - xir_graph_t g = xir_graph_deserialize(argv[2]); - printf("graph name is %s\n", xir_graph_get_name(g)); - xir_graph_destroy(g); - return 0; -} - -int hello_attr(int argc, char* argv[]) { - xir_attrs_t a = xir_attrs_create(); - xir_attrs_set_string(a, "hello", "world"); - if (xir_attrs_has_string(a, "hello")) { - printf("has attrs(\"hello\") %s\n", - xir_attrs_has_attr(a, "hello") ? "true" : "false"); - } else { - assert(0 && "logical error: xir_attrs_has_string should return true"); - } - printf("hello %s\n", xir_attrs_get_string(a, "hello")); - /* test string array */ - xir_attrs_set_vec_string(a, "numbers", 0, "zero"); - // xir_attrs_set_vec_string(a, "numbers", 1, "one"); - xir_attrs_set_vec_string(a, "numbers", 2, "two"); - printf("has numbers? %s\n", - xir_attrs_has_vec_string(a, "numbers") ? "true" : "false"); - size_t n_of_numbers = xir_attrs_get_vec_size_string(a, "numbers"); - for (size_t i = 0; i < n_of_numbers; ++i) { - printf("numbers[%d] = %s\n", (int)i, - xir_attrs_get_vec_string(a, "numbers", i)); - } - /* test string mapping */ - xir_attrs_set_map_string(a, "month", "Jan", "Januray"); - xir_attrs_set_map_string(a, "month", "Feb", "Feberay"); - size_t n_of_months = xir_attrs_get_map_size_string(a, "month"); - const char* keys[n_of_months]; - xir_attrs_get_map_keys_string(a, "month", keys); - for (size_t i = 0; i < n_of_months; ++i) { - printf("%s => %s\n", keys[i], - xir_attrs_get_map_string(a, "month", keys[i])); - } - /* test map_vec_string */ - xir_attrs_set_map_vec_string(a, "items", "colors", 0, "red"); - xir_attrs_set_map_vec_string(a, "items", "colors", 1, "green"); - xir_attrs_set_map_vec_string(a, "items", "colors", 2, "blue"); - xir_attrs_set_map_vec_string(a, "items", "shapes", 0, "circle"); - xir_attrs_set_map_vec_string(a, "items", "shapes", 1, "line"); - xir_attrs_set_map_vec_string(a, "items", "shapes", 2, "cross"); - printf("has items ? %s\n", - xir_attrs_has_map_vec_string(a, "items") ? "true" : "false"); - size_t n_of_items = xir_attrs_get_map_vec_msize_string(a, "items"); - const char* item_keys[n_of_items]; - xir_attrs_get_map_vec_keys_string(a, "items", item_keys); - for (size_t i = 0; i < n_of_items; ++i) { - size_t n_of_x = - xir_attrs_get_map_vec_vsize_string(a, "items", item_keys[i]); - printf("%s => [", item_keys[i]); - for (size_t j = 0; j < n_of_x; ++j) { - if (j != 0) { - printf(","); - } - printf("'%s'", xir_attrs_get_map_vec_string(a, "items", item_keys[i], j)); - } - printf("]\n"); - } - return 0; -} - -int show_all_keys(int argc, char* argv[]) { - xir_attrs_t a = xir_attrs_create(); - xir_attrs_set_string(a, "one", "1"); - xir_attrs_set_string(a, "two", "2"); - xir_attrs_set_string(a, "three", "3"); - xir_attrs_set_string(a, "four", "4"); - for (size_t i = 0u; i < xir_attrs_get_num_of_keys(a); ++i) { - printf("%s => %s\n", xir_attrs_get_key(a, i), - xir_attrs_get_string(a, xir_attrs_get_key(a, i))); - } - return 0; -} - -xir_graph_t create_simple_graph() { - xir_graph_t graph = xir_graph_create("graph_test"); - int data_dims[] = {1, 1, 1}; - xir_op_t data_op = xir_graph_add_op( - graph, "data", "data", xir_attrs_create(), NULL, 0, - xir_tensor_create("data", data_dims, - sizeof(data_dims) / sizeof(data_dims[0]), XIR_FLOAT, - 32), - NULL); - (void)(data_op); - return graph; -} - -int simple_graph(int argc, char* argv[]) { - xir_graph_t graph = create_simple_graph(); - printf("graph name is %s\n", xir_graph_get_name(graph)); - xir_graph_destroy(graph); - return 0; -} - -int show_tensor(int argc, char* argv[]) { - int dims[] = {4, 480, 360, 3}; - xir_tensor_t tensor = xir_tensor_create( - "tensor_test", dims, sizeof(dims) / sizeof(dims[0]), XIR_UINT, 8); - printf("tensor name is %s\t tensor dims[ ", xir_tensor_get_name(tensor)); - int32_t dim_num = xir_tensor_get_dim_num(tensor); - for (int32_t i = 0; i < dim_num; i++) { - printf("%d ", xir_tensor_get_dim_size(tensor, i)); - } - printf("]\t tensor bit_width %d\t tensor_data_type %d\n", - xir_tensor_get_bit_width(tensor), xir_tensor_get_data_type(tensor)); - - xir_tensor_destroy(tensor); - return 0; -} - -int show_subgraph_children(int argc, char* argv[]) { - xir_graph_t g = xir_graph_deserialize(argv[2]); - printf("graph name is %s\n", xir_graph_get_name(g)); - - xir_subgraph_t root = xir_graph_get_root_subgraph(g); - printf("root subgraph name is %s\n", xir_subgraph_get_name(root)); - - int32_t children_num = xir_subgraph_get_children_num(root); - printf("root subgraph children num is %d\n", children_num); - - for (int32_t i = 0; i < children_num; i++) { - xir_subgraph_t subgraph = xir_subgraph_get_child(root, i); - printf("children %d name is %s\n", i, xir_subgraph_get_name(subgraph)); - } - printf("=== topological_sort_children===\n"); - xir_subgraph_t children[children_num]; - xir_subgraph_children_topological_sort(root, children); - for (int32_t i = 0; i < children_num; i++) { - printf("children %d name is %s\n", i, xir_subgraph_get_name(children[i])); - } - // free(children); - xir_graph_destroy(g); - return 0; -} - -int main(int argc, char* argv[]) { - const char* test_case = argv[1]; - if (strcmp(test_case, "show_name") == 0) { - return show_name(argc, argv); - } else if (strcmp(test_case, "hello_attr") == 0) { - return hello_attr(argc, argv); - } else if (strcmp(test_case, "show_all_keys") == 0) { - return show_all_keys(argc, argv); - } else if (strcmp(test_case, "simple_graph") == 0) { - return simple_graph(argc, argv); - } else if (strcmp(test_case, "show_tensor") == 0) { - return show_tensor(argc, argv); - } else if (strcmp(test_case, "show_subgraph_children") == 0) { - return show_subgraph_children(argc, argv); - } else { - printf("unknown test case %s\n", test_case); - } - return 0; -} - -/* Local Variables: */ -/* mode:c */ -/* c-basic-offset: 2 */ -/* coding: utf-8-unix */ -/* End: */ diff -pruN 1.4-2/test/test_md5sum.cpp 2.5-1/test/test_md5sum.cpp --- 1.4-2/test/test_md5sum.cpp 1970-01-01 00:00:00.000000000 +0000 +++ 2.5-1/test/test_md5sum.cpp 2022-06-15 07:33:00.000000000 +0000 @@ -0,0 +1,44 @@ +/* + * Copyright 2019 Xilinx Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "UniLog/UniLog.hpp" +#include "xir/util/tool_function.hpp" + +// helper function to add a conv +xir::Op* add_conv(std::string name, std::shared_ptr graph, + xir::Op* input); +// helper function to creat a graph +std::shared_ptr create_test_graph(); + +static uint32_t get_file_size(std::string file_name) { + std::ifstream infile(file_name, std::ios::binary | std::ios::ate); + UNI_LOG_CHECK(infile.is_open(), VAIEDIFF_BAD_FILE) + << "Cannot open file " << file_name; + return infile.tellg(); +} + +int main(int, char* argv[]) { + UniLog::Initial(argv[0], UNI_LOG_STD, UNI_LOG_LEVEL_INFO, + UNI_LOG_STD_LEVEL_INFO); + auto buf = std::vector((size_t)get_file_size(__FILE__)); + CHECK(std::ifstream(__FILE__).read(&buf[0], buf.size()).good()); + UNI_LOG_INFO << "md5sum " << __FILE__ << " = " + << xir::get_md5_of_buffer(&buf[0], buf.size()); + + UNI_LOG_INFO << "md5sum " << __FILE__ << " = " + << xir::get_md5_of_file(__FILE__); + return 0; +} diff -pruN 1.4-2/test/test_subgraph_tensors_sort.cpp 2.5-1/test/test_subgraph_tensors_sort.cpp --- 1.4-2/test/test_subgraph_tensors_sort.cpp 1970-01-01 00:00:00.000000000 +0000 +++ 2.5-1/test/test_subgraph_tensors_sort.cpp 2022-06-15 07:33:00.000000000 +0000 @@ -0,0 +1,84 @@ +/* + * Copyright 2019 Xilinx Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "UniLog/UniLog.hpp" +#include "xir/graph/graph.hpp" + +std::shared_ptr create_test_graph(); + +int main(int, char* argv[]) { + UniLog::Initial(argv[0], UNI_LOG_STD, UNI_LOG_LEVEL_INFO, + UNI_LOG_STD_LEVEL_INFO); + auto graph = create_test_graph(); + auto root = graph->get_root_subgraph(); + root->create_children(); + auto child = root->merge_children({root->find_op("conv0")}); + for(auto tensors_i : child->get_input_tensors()) + UNI_LOG_INFO << "Conv0 Input Tensors: " << tensors_i->get_name(); + for(auto tensors_sort_i : child->get_sorted_input_tensors()) + UNI_LOG_INFO << "Conv0 Input Sorted Tensors: " << tensors_sort_i->get_name(); + + for(auto tensors_o : root->get_output_tensors()) + UNI_LOG_INFO << "Graph Output Tensors: " << tensors_o->get_name(); + for(auto tensors_sort_o : root->get_sorted_output_tensors()) + UNI_LOG_INFO << "Graph Output Sorted Tensors: " << tensors_sort_o->get_name(); + return 0; +} + +xir::Op* add_conv(std::string name, std::shared_ptr graph, + xir::Op* input) { + auto attrs = xir::Attrs::create(); + attrs->set_attr>("kernel", {3, 3}); + attrs->set_attr>("stride", {3, 3}); + attrs->set_attr("pad_mode", "SAME"); + attrs->set_attr>("pad", {1, 1, 1, 1}); + + auto weights_attrs = xir::Attrs::create(); + auto weights_data = std::vector(3 * 3 * 3 * 3 * 4, 1); + weights_attrs->set_attr>("shape", {3, 3, 3, 3}) + ->set_attr("data_type", "FLOAT32") + ->set_attr>("data", weights_data); + auto weights = graph->add_op(name + "_w", // + "const", // + std::move(weights_attrs), // + {}); + auto bias_attrs = xir::Attrs::create(); + auto bias_data = std::vector(3 * 4, 1); + bias_attrs->set_attr>("shape", {3}) + ->set_attr("data_type", "FLOAT32") + ->set_attr>("data", weights_data); + auto bias = graph->add_op(name + "_b", // + "const", // + std::move(bias_attrs), // + {}); + return graph->add_op( + name, // + "conv2d", // + std::move(attrs), // + {{"weights", {weights}}, {"bias", {bias}}, {"input", {input}}}); +} + +std::shared_ptr create_test_graph() { + std::shared_ptr graph = xir::Graph::create("graph_test"); + auto data_attrs = xir::Attrs::create(); + data_attrs->set_attr>("shape", {1, 224, 224, 3}) + ->set_attr("data_type", "FLOAT32"); + auto data = graph->add_op("data0","data",std::move(data_attrs),{}); + add_conv("conv2", graph, data); + add_conv("conv1", graph, data); + add_conv("conv0", graph, data); + return graph; +} diff -pruN 1.4-2/tools/CMakeLists.txt 2.5-1/tools/CMakeLists.txt --- 1.4-2/tools/CMakeLists.txt 2021-07-27 05:41:52.000000000 +0000 +++ 2.5-1/tools/CMakeLists.txt 2022-06-15 07:33:00.000000000 +0000 @@ -1,20 +1,21 @@ # # Copyright 2019 Xilinx Inc. # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. # -add_executable (xir_util +add_executable( + xir_util xir_util.hpp xir_util.cpp cmd_dump_reg.hpp @@ -30,11 +31,10 @@ add_executable (xir_util cmd_dump_code.hpp cmd_dump_code.cpp cmd_graph.hpp - cmd_graph.cpp - ) -target_link_libraries (xir_util xir protobuf::libprotobuf crypto) -target_include_directories (xir_util PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/../src/xir) -set_target_properties (xir_util PROPERTIES - OUTPUT_NAME xir) + cmd_graph.cpp) +target_link_libraries(xir_util xir protobuf::libprotobuf unilog::unilog) +target_include_directories(xir_util + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/../src/xir) +set_target_properties(xir_util PROPERTIES OUTPUT_NAME xir) -install (TARGETS xir_util DESTINATION bin) +install(TARGETS xir_util DESTINATION bin) diff -pruN 1.4-2/tools/cmd_dump_code.cpp 2.5-1/tools/cmd_dump_code.cpp --- 1.4-2/tools/cmd_dump_code.cpp 2021-07-27 05:41:52.000000000 +0000 +++ 2.5-1/tools/cmd_dump_code.cpp 2022-06-15 07:33:00.000000000 +0000 @@ -16,13 +16,11 @@ #include "cmd_dump_code.hpp" -#include #include #include -#include #include -#include +#include #include #include #include @@ -31,19 +29,12 @@ using namespace std; static void mkdir_minus_p(const std::string& dirname) { - struct stat st = {0}; - if (stat(dirname.c_str(), &st) == -1) { - PCHECK(mkdir(dirname.c_str(), 0777) == 0) - << "mkdir error; dirname=" << dirname; - } - PCHECK(stat(dirname.c_str(), &st) == 0) - << "stat dir error: dirname=" << dirname; - CHECK(S_ISDIR(st.st_mode)) << "error not a directory: dirname=" << dirname; + CHECK(std::filesystem::create_directories(dirname)) + << "cannot create directories: " << dirname; } static bool is_exist_path(const std::string& filename) { - struct stat buffer; - return (stat(filename.c_str(), &buffer) == 0 && S_ISDIR(buffer.st_mode)); + return std::filesystem::exists(filename); } static std::string get_parent_path(const std::string& path) { diff -pruN 1.4-2/tools/cmd_dump_reg.cpp 2.5-1/tools/cmd_dump_reg.cpp --- 1.4-2/tools/cmd_dump_reg.cpp 2021-07-27 05:41:52.000000000 +0000 +++ 2.5-1/tools/cmd_dump_reg.cpp 2022-06-15 07:33:00.000000000 +0000 @@ -16,11 +16,11 @@ #include "cmd_dump_reg.hpp" -#include - #include #include #include + +#include "xir/util/tool_function.hpp" using namespace std; CmdDumpReg::CmdDumpReg(const std::string& name) : Cmd(name) {} @@ -34,16 +34,6 @@ static std::string get_device(const xir: return ret; } -static std::string md5sum(const char* val, size_t s) { - std::vector result((size_t)MD5_DIGEST_LENGTH, '0'); - std::ostringstream str; - MD5((const unsigned char*)&val[0], s, (unsigned char*)&result[0]); - for (const auto x : result) { - str << std::hex << std::setfill('0') << std::setw(2) << ((unsigned int)x); - } - return str.str(); -} - static std::string reg_info(const xir::Subgraph* s, int n) { std::ostringstream str; auto reg_id_to_context_type = @@ -71,7 +61,8 @@ static std::string reg_info(const xir::S str << indent(n + 1) << "reg_id_to_parameter_value:" << '\n'; for (auto& r : reg_id_to_parameter_value) { str << indent(n + 1) << r.first << " => " << r.second.size() - << " bytes md5sum= " << md5sum(&r.second[0], r.second.size()) << '\n'; + << " bytes md5sum= " + << xir::get_md5_of_buffer(&r.second[0], r.second.size()) << '\n'; } auto reg_id_to_size = s->get_attr>("reg_id_to_size"); diff -pruN 1.4-2/tools/cmd_dump_txt.cpp 2.5-1/tools/cmd_dump_txt.cpp --- 1.4-2/tools/cmd_dump_txt.cpp 2021-07-27 05:41:52.000000000 +0000 +++ 2.5-1/tools/cmd_dump_txt.cpp 2022-06-15 07:33:00.000000000 +0000 @@ -15,10 +15,9 @@ */ #include "cmd_dump_txt.hpp" - +// must include this first to define XIR_DLLSPEC; #include #include -#include #include #include @@ -26,6 +25,8 @@ #include #include "graph_proto_v2.pb.h" +#include "xir/XirExport.hpp" +#include "xir/util/tool_function.hpp" static inline std::string xxd(const unsigned char* p, int size, int column, int group) { std::ostringstream str; @@ -47,15 +48,6 @@ static inline std::string xxd(const unsi return str.str(); } -static std::string md5sum(const std::string& val) { - std::vector result((size_t)MD5_DIGEST_LENGTH, '0'); - std::ostringstream str; - MD5((const unsigned char*)&val[0], val.size(), (unsigned char*)&result[0]); - for (const auto x : result) { - str << std::hex << std::setfill('0') << std::setw(2) << ((unsigned int)x); - } - return str.str(); -} static std::string dump_string(const std::string& val, const std::string& md5) { std::ostringstream str; const std::string::size_type max_size = 160u; @@ -73,7 +65,7 @@ struct MyPrinter : public google::protob virtual void PrintBytes(const std::string& val, google::protobuf::TextFormat::BaseTextGenerator* generator) const override { - auto md5value = md5sum(val); + auto md5value = xir::get_md5_of_buffer(&val[0], val.size()); std::string n = dump_string(val, md5value); generator->PrintString(n); if (dump_) { @@ -108,7 +100,7 @@ std::string CmdDumpTxt::help() const { << "\n\t" << "e.g. xir " << get_name() << " a.xmodel a.txt" << "\n\t" - << "when is missing, it dumps to standard output." << endl; + << "when is missing, it dumps to standard output." << std::endl; return str.str(); } diff -pruN 1.4-2/tools/cmd_graph.cpp 2.5-1/tools/cmd_graph.cpp --- 1.4-2/tools/cmd_graph.cpp 2021-07-27 05:41:52.000000000 +0000 +++ 2.5-1/tools/cmd_graph.cpp 2022-06-15 07:33:00.000000000 +0000 @@ -103,13 +103,13 @@ struct is_cout_able< : public std::true_type {}; template -constexpr auto is_cout_able_v = is_cout_able::value; +constexpr bool is_cout_able_v = is_cout_able::value; template std::enable_if_t && is_cout_able_v, std::ostream&> operator<<(std::ostream& out, const std::map& v) { out << "{"; - for (const auto x : v) { + for (const auto& x : v) { out << "\n\t\"" << x.first << "\" = " << x.second; } out << "\n}"; diff -pruN 1.4-2/tools/xir_util.cpp 2.5-1/tools/xir_util.cpp --- 1.4-2/tools/xir_util.cpp 2021-07-27 05:41:52.000000000 +0000 +++ 2.5-1/tools/xir_util.cpp 2022-06-15 07:33:00.000000000 +0000 @@ -14,7 +14,7 @@ * limitations under the License. */ #include "./xir_util.hpp" - +#include #include #include #include diff -pruN 1.4-2/tools/xir_util.hpp 2.5-1/tools/xir_util.hpp --- 1.4-2/tools/xir_util.hpp 2021-07-27 05:41:52.000000000 +0000 +++ 2.5-1/tools/xir_util.hpp 2022-06-15 07:33:00.000000000 +0000 @@ -46,7 +46,7 @@ static inline std::ostream& operator<<(s const std::set& v) { int c = 0; out << "["; - for (const auto x : v) { + for (const auto& x : v) { if (c++ != 0) { out << ","; }