PYTHON-1521 - Use FNV-1a for ObjectId machine bytes

This commit is contained in:
Bernie Hackett 2018-05-31 12:26:53 -07:00
parent 714577c83e
commit 61850357a0
2 changed files with 437 additions and 11 deletions

View File

@ -19,7 +19,6 @@
import binascii
import calendar
import datetime
import hashlib
import os
import random
import socket
@ -31,19 +30,35 @@ from bson.errors import InvalidId
from bson.py3compat import PY3, bytes_from_hex, string_type, text_type
from bson.tz_util import utc
if PY3:
_ord = lambda x: x
else:
_ord = ord
# http://isthe.com/chongo/tech/comp/fnv/index.html#FNV-1a
def _fnv_1a_24(data, _ord=_ord):
"""FNV-1a 24 bit hash"""
# http://www.isthe.com/chongo/tech/comp/fnv/index.html#xor-fold
# Start with FNV-1a 32 bit.
hash_size = 2 ** 32
fnv_32_prime = 16777619
fnv_1a_hash = 2166136261 # 32-bit FNV-1 offset basis
for elt in data:
fnv_1a_hash = fnv_1a_hash ^ _ord(elt)
fnv_1a_hash = (fnv_1a_hash * fnv_32_prime) % hash_size
# xor-fold the result to 24 bit.
return (fnv_1a_hash >> 24) ^ (fnv_1a_hash & 0xffffff)
def _machine_bytes():
"""Get the machine portion of an ObjectId.
"""
machine_hash = hashlib.md5()
if PY3:
# gethostname() returns a unicode string in python 3.x
# while update() requires a byte string.
machine_hash.update(socket.gethostname().encode())
else:
# Calling encode() here will fail with non-ascii hostnames
machine_hash.update(socket.gethostname())
return machine_hash.digest()[0:3]
# gethostname() returns a unicode string in python 3.x
# We only need 3 bytes, and _fnv_1a_24 returns a 24 bit integer.
# Remove the padding byte.
return struct.pack("<I", _fnv_1a_24(socket.gethostname().encode()))[:3]
def _raise_invalid_id(oid):

View File

@ -16,12 +16,13 @@
import datetime
import pickle
import struct
import sys
sys.path[0:0] = [""]
from bson.errors import InvalidId
from bson.objectid import ObjectId
from bson.objectid import ObjectId, _fnv_1a_24
from bson.py3compat import PY3, _unicode
from bson.tz_util import (FixedOffset,
utc)
@ -165,5 +166,415 @@ class TestObjectId(unittest.TestCase):
self.assertTrue(ObjectId.is_valid(b"123456789012"))
self.assertTrue(ObjectId.is_valid("123456789012123456789012"))
# Landon Curt Noll's test suite for FNV-1a32
# http://isthe.com/chongo/src/fnv/test_fnv.c
TEST = lambda x: x
TEST0 = lambda x: x + b"\x00"
R10 = lambda x: x * 10
R500 = lambda x: x * 500
fnv_test_str = [
TEST(b""),
TEST(b"a"),
TEST(b"b"),
TEST(b"c"),
TEST(b"d"),
TEST(b"e"),
TEST(b"f"),
TEST(b"fo"),
TEST(b"foo"),
TEST(b"foob"),
TEST(b"fooba"),
TEST(b"foobar"),
TEST0(b""),
TEST0(b"a"),
TEST0(b"b"),
TEST0(b"c"),
TEST0(b"d"),
TEST0(b"e"),
TEST0(b"f"),
TEST0(b"fo"),
TEST0(b"foo"),
TEST0(b"foob"),
TEST0(b"fooba"),
TEST0(b"foobar"),
TEST(b"ch"),
TEST(b"cho"),
TEST(b"chon"),
TEST(b"chong"),
TEST(b"chongo"),
TEST(b"chongo "),
TEST(b"chongo w"),
TEST(b"chongo wa"),
TEST(b"chongo was"),
TEST(b"chongo was "),
TEST(b"chongo was h"),
TEST(b"chongo was he"),
TEST(b"chongo was her"),
TEST(b"chongo was here"),
TEST(b"chongo was here!"),
TEST(b"chongo was here!\n"),
TEST0(b"ch"),
TEST0(b"cho"),
TEST0(b"chon"),
TEST0(b"chong"),
TEST0(b"chongo"),
TEST0(b"chongo "),
TEST0(b"chongo w"),
TEST0(b"chongo wa"),
TEST0(b"chongo was"),
TEST0(b"chongo was "),
TEST0(b"chongo was h"),
TEST0(b"chongo was he"),
TEST0(b"chongo was her"),
TEST0(b"chongo was here"),
TEST0(b"chongo was here!"),
TEST0(b"chongo was here!\n"),
TEST(b"cu"),
TEST(b"cur"),
TEST(b"curd"),
TEST(b"curds"),
TEST(b"curds "),
TEST(b"curds a"),
TEST(b"curds an"),
TEST(b"curds and"),
TEST(b"curds and "),
TEST(b"curds and w"),
TEST(b"curds and wh"),
TEST(b"curds and whe"),
TEST(b"curds and whey"),
TEST(b"curds and whey\n"),
TEST0(b"cu"),
TEST0(b"cur"),
TEST0(b"curd"),
TEST0(b"curds"),
TEST0(b"curds "),
TEST0(b"curds a"),
TEST0(b"curds an"),
TEST0(b"curds and"),
TEST0(b"curds and "),
TEST0(b"curds and w"),
TEST0(b"curds and wh"),
TEST0(b"curds and whe"),
TEST0(b"curds and whey"),
TEST0(b"curds and whey\n"),
TEST(b"hi"), TEST0(b"hi"),
TEST(b"hello"), TEST0(b"hello"),
TEST(b"\xff\x00\x00\x01"), TEST(b"\x01\x00\x00\xff"),
TEST(b"\xff\x00\x00\x02"), TEST(b"\x02\x00\x00\xff"),
TEST(b"\xff\x00\x00\x03"), TEST(b"\x03\x00\x00\xff"),
TEST(b"\xff\x00\x00\x04"), TEST(b"\x04\x00\x00\xff"),
TEST(b"\x40\x51\x4e\x44"), TEST(b"\x44\x4e\x51\x40"),
TEST(b"\x40\x51\x4e\x4a"), TEST(b"\x4a\x4e\x51\x40"),
TEST(b"\x40\x51\x4e\x54"), TEST(b"\x54\x4e\x51\x40"),
TEST(b"127.0.0.1"), TEST0(b"127.0.0.1"),
TEST(b"127.0.0.2"), TEST0(b"127.0.0.2"),
TEST(b"127.0.0.3"), TEST0(b"127.0.0.3"),
TEST(b"64.81.78.68"), TEST0(b"64.81.78.68"),
TEST(b"64.81.78.74"), TEST0(b"64.81.78.74"),
TEST(b"64.81.78.84"), TEST0(b"64.81.78.84"),
TEST(b"feedface"), TEST0(b"feedface"),
TEST(b"feedfacedaffdeed"), TEST0(b"feedfacedaffdeed"),
TEST(b"feedfacedeadbeef"), TEST0(b"feedfacedeadbeef"),
TEST(b"line 1\nline 2\nline 3"),
TEST(b"chongo <Landon Curt Noll> /\\../\\"),
TEST0(b"chongo <Landon Curt Noll> /\\../\\"),
TEST(b"chongo (Landon Curt Noll) /\\../\\"),
TEST0(b"chongo (Landon Curt Noll) /\\../\\"),
TEST(b"http://antwrp.gsfc.nasa.gov/apod/astropix.html"),
TEST(b"http://en.wikipedia.org/wiki/Fowler_Noll_Vo_hash"),
TEST(b"http://epod.usra.edu/"),
TEST(b"http://exoplanet.eu/"),
TEST(b"http://hvo.wr.usgs.gov/cam3/"),
TEST(b"http://hvo.wr.usgs.gov/cams/HMcam/"),
TEST(b"http://hvo.wr.usgs.gov/kilauea/update/deformation.html"),
TEST(b"http://hvo.wr.usgs.gov/kilauea/update/images.html"),
TEST(b"http://hvo.wr.usgs.gov/kilauea/update/maps.html"),
TEST(b"http://hvo.wr.usgs.gov/volcanowatch/current_issue.html"),
TEST(b"http://neo.jpl.nasa.gov/risk/"),
TEST(b"http://norvig.com/21-days.html"),
TEST(b"http://primes.utm.edu/curios/home.php"),
TEST(b"http://slashdot.org/"),
TEST(b"http://tux.wr.usgs.gov/Maps/155.25-19.5.html"),
TEST(b"http://volcano.wr.usgs.gov/kilaueastatus.php"),
TEST(b"http://www.avo.alaska.edu/activity/Redoubt.php"),
TEST(b"http://www.dilbert.com/fast/"),
TEST(b"http://www.fourmilab.ch/gravitation/orbits/"),
TEST(b"http://www.fpoa.net/"),
TEST(b"http://www.ioccc.org/index.html"),
TEST(b"http://www.isthe.com/cgi-bin/number.cgi"),
TEST(b"http://www.isthe.com/chongo/bio.html"),
TEST(b"http://www.isthe.com/chongo/index.html"),
TEST(b"http://www.isthe.com/chongo/src/calc/lucas-calc"),
TEST(b"http://www.isthe.com/chongo/tech/astro/venus2004.html"),
TEST(b"http://www.isthe.com/chongo/tech/astro/vita.html"),
TEST(b"http://www.isthe.com/chongo/tech/comp/c/expert.html"),
TEST(b"http://www.isthe.com/chongo/tech/comp/calc/index.html"),
TEST(b"http://www.isthe.com/chongo/tech/comp/fnv/index.html"),
TEST(b"http://www.isthe.com/chongo/tech/math/number/howhigh.html"),
TEST(b"http://www.isthe.com/chongo/tech/math/number/number.html"),
TEST(b"http://www.isthe.com/chongo/tech/math/prime/mersenne.html"),
TEST(b"http://www.isthe.com/chongo/tech/math/prime/mersenne.html#largest"),
TEST(b"http://www.lavarnd.org/cgi-bin/corpspeak.cgi"),
TEST(b"http://www.lavarnd.org/cgi-bin/haiku.cgi"),
TEST(b"http://www.lavarnd.org/cgi-bin/rand-none.cgi"),
TEST(b"http://www.lavarnd.org/cgi-bin/randdist.cgi"),
TEST(b"http://www.lavarnd.org/index.html"),
TEST(b"http://www.lavarnd.org/what/nist-test.html"),
TEST(b"http://www.macosxhints.com/"),
TEST(b"http://www.mellis.com/"),
TEST(b"http://www.nature.nps.gov/air/webcams/parks/havoso2alert/havoalert.cfm"),
TEST(b"http://www.nature.nps.gov/air/webcams/parks/havoso2alert/timelines_24.cfm"),
TEST(b"http://www.paulnoll.com/"),
TEST(b"http://www.pepysdiary.com/"),
TEST(b"http://www.sciencenews.org/index/home/activity/view"),
TEST(b"http://www.skyandtelescope.com/"),
TEST(b"http://www.sput.nl/~rob/sirius.html"),
TEST(b"http://www.systemexperts.com/"),
TEST(b"http://www.tq-international.com/phpBB3/index.php"),
TEST(b"http://www.travelquesttours.com/index.htm"),
TEST(b"http://www.wunderground.com/global/stations/89606.html"),
TEST(R10(b"21701")),
TEST(R10(b"M21701")),
TEST(R10(b"2^21701-1")),
TEST(R10(b"\x54\xc5")),
TEST(R10(b"\xc5\x54")),
TEST(R10(b"23209")),
TEST(R10(b"M23209")),
TEST(R10(b"2^23209-1")),
TEST(R10(b"\x5a\xa9")),
TEST(R10(b"\xa9\x5a")),
TEST(R10(b"391581216093")),
TEST(R10(b"391581*2^216093-1")),
TEST(R10(b"\x05\xf9\x9d\x03\x4c\x81")),
TEST(R10(b"FEDCBA9876543210")),
TEST(R10(b"\xfe\xdc\xba\x98\x76\x54\x32\x10")),
TEST(R10(b"EFCDAB8967452301")),
TEST(R10(b"\xef\xcd\xab\x89\x67\x45\x23\x01")),
TEST(R10(b"0123456789ABCDEF")),
TEST(R10(b"\x01\x23\x45\x67\x89\xab\xcd\xef")),
TEST(R10(b"1032547698BADCFE")),
TEST(R10(b"\x10\x32\x54\x76\x98\xba\xdc\xfe")),
TEST(R500(b"\x00")),
TEST(R500(b"\x07")),
TEST(R500(b"~")),
TEST(R500(b"\x7f"))
]
fnv1a_24_vector = {
fnv_test_str[0]: 0x1c9d44,
fnv_test_str[1]: 0x0c29c8,
fnv_test_str[2]: 0x0c2d02,
fnv_test_str[3]: 0x0c2cb4,
fnv_test_str[4]: 0x0c2492,
fnv_test_str[5]: 0x0c2200,
fnv_test_str[6]: 0x0c277a,
fnv_test_str[7]: 0x22e820,
fnv_test_str[8]: 0xf37e7e,
fnv_test_str[9]: 0x5076d0,
fnv_test_str[10]: 0xaaa1b3,
fnv_test_str[11]: 0x9cf9d7,
fnv_test_str[12]: 0x0c5d1a,
fnv_test_str[13]: 0x24d06f,
fnv_test_str[14]: 0x2c3fe2,
fnv_test_str[15]: 0x29c561,
fnv_test_str[16]: 0x1d61b0,
fnv_test_str[17]: 0x1ae633,
fnv_test_str[18]: 0x2255de,
fnv_test_str[19]: 0xf39f58,
fnv_test_str[20]: 0x50ac14,
fnv_test_str[21]: 0xab3aa7,
fnv_test_str[22]: 0x9c4c6f,
fnv_test_str[23]: 0x1c9eb4,
fnv_test_str[24]: 0x299f11,
fnv_test_str[25]: 0x85801c,
fnv_test_str[26]: 0x29778b,
fnv_test_str[27]: 0x46b985,
fnv_test_str[28]: 0x564ec0,
fnv_test_str[29]: 0xdd5c0c,
fnv_test_str[30]: 0x77eded,
fnv_test_str[31]: 0xca9677,
fnv_test_str[32]: 0xeb9b9a,
fnv_test_str[33]: 0xe67a30,
fnv_test_str[34]: 0xd32f6a,
fnv_test_str[35]: 0x743fc8,
fnv_test_str[36]: 0x006376,
fnv_test_str[37]: 0x9c99cb,
fnv_test_str[38]: 0x8524b9,
fnv_test_str[39]: 0x993001,
fnv_test_str[40]: 0x85c7d6,
fnv_test_str[41]: 0x29fe8b,
fnv_test_str[42]: 0x469249,
fnv_test_str[43]: 0x56698e,
fnv_test_str[44]: 0xdd8e4c,
fnv_test_str[45]: 0x787611,
fnv_test_str[46]: 0xca6243,
fnv_test_str[47]: 0xeaf0e4,
fnv_test_str[48]: 0xe648b0,
fnv_test_str[49]: 0xd355aa,
fnv_test_str[50]: 0x740522,
fnv_test_str[51]: 0x004d4e,
fnv_test_str[52]: 0x9c09a7,
fnv_test_str[53]: 0x84f129,
fnv_test_str[54]: 0x993a9d,
fnv_test_str[55]: 0x27dfcd,
fnv_test_str[56]: 0x298129,
fnv_test_str[57]: 0x5637c9,
fnv_test_str[58]: 0xb9140f,
fnv_test_str[59]: 0x5bf5a7,
fnv_test_str[60]: 0xc42805,
fnv_test_str[61]: 0xcc0e97,
fnv_test_str[62]: 0x3b4c5d,
fnv_test_str[63]: 0x59f0a7,
fnv_test_str[64]: 0x94de0b,
fnv_test_str[65]: 0x5a0a72,
fnv_test_str[66]: 0xbee56f,
fnv_test_str[67]: 0x8363fd,
fnv_test_str[68]: 0xd5346c,
fnv_test_str[69]: 0xa14715,
fnv_test_str[70]: 0x56b1b5,
fnv_test_str[71]: 0xb8e81f,
fnv_test_str[72]: 0x5b4a33,
fnv_test_str[73]: 0xc3f6c5,
fnv_test_str[74]: 0xcc3f23,
fnv_test_str[75]: 0x3b0a59,
fnv_test_str[76]: 0x59c467,
fnv_test_str[77]: 0x9510cb,
fnv_test_str[78]: 0x59bdc4,
fnv_test_str[79]: 0xbf0b0f,
fnv_test_str[80]: 0x83ff3d,
fnv_test_str[81]: 0xd54252,
fnv_test_str[82]: 0xa156e9,
fnv_test_str[83]: 0xe2d780,
fnv_test_str[84]: 0x3af6f2,
fnv_test_str[85]: 0xd234c0,
fnv_test_str[86]: 0x9f2ce4,
fnv_test_str[87]: 0x935133,
fnv_test_str[88]: 0x8fb8a9,
fnv_test_str[89]: 0x69f34b,
fnv_test_str[90]: 0x8fb375,
fnv_test_str[91]: 0xef1266,
fnv_test_str[92]: 0x8fb585,
fnv_test_str[93]: 0xc3bfd1,
fnv_test_str[94]: 0x8fb031,
fnv_test_str[95]: 0xe4d46f,
fnv_test_str[96]: 0x17906a,
fnv_test_str[97]: 0x0bfece,
fnv_test_str[97]: 0x0bfece,
fnv_test_str[98]: 0x178d02,
fnv_test_str[99]: 0xaddad9,
fnv_test_str[100]: 0x17a9ca,
fnv_test_str[101]: 0x2633a1,
fnv_test_str[102]: 0xa3d116,
fnv_test_str[103]: 0xe2328d,
fnv_test_str[104]: 0xa3cf8c,
fnv_test_str[105]: 0xdfb740,
fnv_test_str[106]: 0xa3cdfe,
fnv_test_str[107]: 0xdd3d03,
fnv_test_str[108]: 0x5636ba,
fnv_test_str[109]: 0xb80830,
fnv_test_str[110]: 0x53e841,
fnv_test_str[111]: 0x16b9a9,
fnv_test_str[112]: 0x5b8948,
fnv_test_str[113]: 0x1a202b,
fnv_test_str[114]: 0x88b139,
fnv_test_str[115]: 0x2f0186,
fnv_test_str[116]: 0x364109,
fnv_test_str[117]: 0x69b55d,
fnv_test_str[118]: 0x7604b9,
fnv_test_str[119]: 0xc8bd3c,
fnv_test_str[120]: 0xb4eab4,
fnv_test_str[121]: 0x4e927c,
fnv_test_str[122]: 0xb140dd,
fnv_test_str[123]: 0x1b25e1,
fnv_test_str[124]: 0xbb59c8,
fnv_test_str[125]: 0x524a34,
fnv_test_str[126]: 0x16ef98,
fnv_test_str[127]: 0x648bd3,
fnv_test_str[128]: 0xa4bc83,
fnv_test_str[129]: 0x53ae47,
fnv_test_str[130]: 0x302859,
fnv_test_str[131]: 0x6deda7,
fnv_test_str[132]: 0x36db15,
fnv_test_str[133]: 0x9d33fc,
fnv_test_str[134]: 0xbb6ce2,
fnv_test_str[135]: 0xf83893,
fnv_test_str[136]: 0x08bf51,
fnv_test_str[137]: 0xcc8e5f,
fnv_test_str[138]: 0xe20f9f,
fnv_test_str[139]: 0xe97f2e,
fnv_test_str[140]: 0x37b27b,
fnv_test_str[141]: 0x9e874a,
fnv_test_str[142]: 0xe63f5a,
fnv_test_str[143]: 0xb50b11,
fnv_test_str[144]: 0xd678e6,
fnv_test_str[145]: 0xd5b723,
fnv_test_str[146]: 0x450bb7,
fnv_test_str[147]: 0x72d79d,
fnv_test_str[148]: 0x06679c,
fnv_test_str[149]: 0x52e15c,
fnv_test_str[150]: 0x9664f7,
fnv_test_str[151]: 0x3258b6,
fnv_test_str[152]: 0xed6ea7,
fnv_test_str[153]: 0x7d7ce2,
fnv_test_str[154]: 0xc71ba1,
fnv_test_str[155]: 0x84f14b,
fnv_test_str[156]: 0x8ecf2e,
fnv_test_str[157]: 0x94f673,
fnv_test_str[158]: 0x970112,
fnv_test_str[159]: 0x6e172a,
fnv_test_str[160]: 0xf8f6e7,
fnv_test_str[161]: 0xf58843,
fnv_test_str[162]: 0x17b6b2,
fnv_test_str[163]: 0xad4cfb,
fnv_test_str[164]: 0x256811,
fnv_test_str[165]: 0xb18dd8,
fnv_test_str[166]: 0x61c153,
fnv_test_str[167]: 0x47d20d,
fnv_test_str[168]: 0x8b689f,
fnv_test_str[169]: 0xd2a40b,
fnv_test_str[170]: 0x549b0a,
fnv_test_str[171]: 0xe1b55b,
fnv_test_str[172]: 0x0cd3d1,
fnv_test_str[173]: 0x471605,
fnv_test_str[174]: 0x5eef10,
fnv_test_str[175]: 0xed3629,
fnv_test_str[176]: 0x624952,
fnv_test_str[177]: 0x9b8688,
fnv_test_str[178]: 0x15e25f,
fnv_test_str[179]: 0xa98d05,
fnv_test_str[180]: 0xdf8bcc,
fnv_test_str[181]: 0x1e9051,
fnv_test_str[182]: 0x3f70db,
fnv_test_str[183]: 0x95aedb,
fnv_test_str[184]: 0xa7f7d7,
fnv_test_str[185]: 0x3bc660,
fnv_test_str[186]: 0x610967,
fnv_test_str[187]: 0x157785,
fnv_test_str[188]: 0x2b2800,
fnv_test_str[189]: 0x8239ef,
fnv_test_str[190]: 0x5869f5,
fnv_test_str[191]: 0x415c76,
fnv_test_str[192]: 0xe4ff6f,
fnv_test_str[193]: 0xb7977d,
fnv_test_str[194]: 0xa43a7b,
fnv_test_str[195]: 0xb3be1e,
fnv_test_str[196]: 0x777aaf,
fnv_test_str[197]: 0x21c38a,
fnv_test_str[198]: 0x9d0839,
fnv_test_str[199]: 0x823d2f,
fnv_test_str[200]: 0xa27250,
fnv_test_str[201]: 0xc5c656,
fnv_test_str[202]: 0x3b0800,
}
class TestFNV1a24(unittest.TestCase):
def test_fnv_1a_24(self):
for key in fnv1a_24_vector:
self.assertEqual(_fnv_1a_24(key), fnv1a_24_vector[key])
if __name__ == "__main__":
unittest.main()