Author: Armin Rigo <ar...@tunes.org>
Branch:
Changeset: r87262:038a80811172
Date: 2016-09-21 13:35 +0200
http://bitbucket.org/pypy/pypy/changeset/038a80811172/
Log: os.stat(): support full-precision nanosecond times in translated
RPython programs, as rbigint numbers.
Before translation, it relies on Python 2.7's os.stat_result. This
means that before translation you get lower-precision rbigints.
diff --git a/rpython/rlib/rposix_stat.py b/rpython/rlib/rposix_stat.py
--- a/rpython/rlib/rposix_stat.py
+++ b/rpython/rlib/rposix_stat.py
@@ -17,7 +17,7 @@
from rpython.rtyper.error import TyperError
from rpython.rlib._os_support import _preferred_traits, string_traits
-from rpython.rlib.objectmodel import specialize
+from rpython.rlib.objectmodel import specialize, we_are_translated
from rpython.rtyper.lltypesystem import lltype, rffi
from rpython.translator.tool.cbuild import ExternalCompilationInfo
from rpython.rlib.rarithmetic import intmask
@@ -51,15 +51,18 @@
("st_uid", lltype.Signed),
("st_gid", lltype.Signed),
("st_size", lltype.SignedLongLong),
- ("st_atime", lltype.Float),
- ("st_mtime", lltype.Float),
- ("st_ctime", lltype.Float),
+ ("st_atime", lltype.SignedLongLong), # integral number of seconds
+ ("st_mtime", lltype.SignedLongLong), #
+ ("st_ctime", lltype.SignedLongLong), #
("st_blksize", lltype.Signed),
("st_blocks", lltype.Signed),
("st_rdev", lltype.Signed),
("st_flags", lltype.Signed),
#("st_gen", lltype.Signed), -- new in CPy 2.5, not implemented
#("st_birthtime", lltype.Float), -- new in CPy 2.5, not implemented
+ ("nsec_atime", lltype.Signed), # number of nanoseconds
+ ("nsec_mtime", lltype.Signed), #
+ ("nsec_ctime", lltype.Signed), #
]
N_INDEXABLE_FIELDS = 10
@@ -79,6 +82,37 @@
("f_namemax", lltype.Signed),
]
+@specialize.arg(1)
+def get_stat_ns_as_bigint(st, name):
+ """'name' is one of the strings "atime", "mtime" or "ctime".
+ Returns a bigint that represents the number of nanoseconds
+ stored inside the RPython-level os.stat_result 'st'.
+
+ Note that when running untranslated, the os.stat_result type
+ is from Python 2.7, which doesn't store more precision than
+ a float anyway. You will only get more after translation.
+ """
+ from rpython.rlib.rbigint import rbigint
+
+ if not we_are_translated():
+ as_float = getattr(st, "st_" + name)
+ return rbigint.fromfloat(as_float * 1e9)
+
+ if name == "atime":
+ i, j = 7, -3
+ elif name == "mtime":
+ i, j = 8, -2
+ elif name == "ctime":
+ i, j = 9, -1
+ else:
+ raise AssertionError(name)
+
+ sec = st[i]
+ nsec = st[j]
+ result = rbigint.fromrarith_int(sec).int_mul(1000000000)
+ result = result.int_add(nsec)
+ return result
+
# ____________________________________________________________
#
@@ -97,7 +131,15 @@
if not s_attr.is_constant():
raise annmodel.AnnotatorError("non-constant attr name in
getattr()")
attrname = s_attr.const
- TYPE = STAT_FIELD_TYPES[attrname]
+ if attrname in ('st_atime', 'st_mtime', 'st_ctime'):
+ # like CPython, in RPython we can read the st_Xtime
+ # attribute and get a floating-point result. We can also
+ # get a full-precision bigint with get_stat_ns_as_bigint().
+ # The floating-point result is computed like a property
+ # by _ll_get_st_Xtime().
+ TYPE = lltype.Float
+ else:
+ TYPE = STAT_FIELD_TYPES[attrname]
return lltype_to_annotation(TYPE)
def _get_rmarshall_support_(self): # for rlib.rmarshal
@@ -105,13 +147,14 @@
# (we ignore the extra values here for simplicity and portability)
def stat_result_reduce(st):
return (st[0], st[1], st[2], st[3], st[4],
- st[5], st[6], st[7], st[8], st[9])
+ st[5], st[6], st[7], st[8], st[9],
+ st[-3], st[-2], st[-1])
def stat_result_recreate(tup):
- return make_stat_result(tup + extra_zeroes)
+ return make_stat_result(tup[:10] + extra_zeroes + tup[-3:])
s_reduced = annmodel.SomeTuple([lltype_to_annotation(TYPE)
for name, TYPE in PORTABLE_STAT_FIELDS])
- extra_zeroes = (0,) * (len(STAT_FIELDS) - len(PORTABLE_STAT_FIELDS))
+ extra_zeroes = (0,) * (len(STAT_FIELDS) - len(PORTABLE_STAT_FIELDS) -
3)
return s_reduced, stat_result_reduce, stat_result_recreate
@@ -119,7 +162,7 @@
def getitem((s_sta, s_int)):
assert s_int.is_constant(), "os.stat()[index]: index must be constant"
index = s_int.const
- assert 0 <= index < N_INDEXABLE_FIELDS, "os.stat()[index] out of range"
+ assert -3 <= index < N_INDEXABLE_FIELDS, "os.stat()[index] out of
range"
name, TYPE = STAT_FIELDS[index]
return lltype_to_annotation(TYPE)
@@ -152,28 +195,61 @@
def rtype_getattr(self, hop):
s_attr = hop.args_s[1]
attr = s_attr.const
+ if attr in ('st_atime', 'st_mtime', 'st_ctime'):
+ ll_func = globals()['_ll_get_' + attr]
+ v_tuple = hop.inputarg(self, arg=0)
+ return hop.gendirectcall(ll_func, v_tuple)
try:
index = self.stat_field_indexes[attr]
except KeyError:
raise TyperError("os.stat().%s: field not available" % (attr,))
return self.redispatch_getfield(hop, index)
+@specialize.memo()
+def _stfld(name):
+ index = STAT_FIELD_NAMES.index(name)
+ return 'item%d' % index
+
+def _ll_get_st_atime(tup):
+ return (float(getattr(tup, _stfld("st_atime"))) +
+ 1E-9 * getattr(tup, _stfld("nsec_atime")))
+
+def _ll_get_st_mtime(tup):
+ return (float(getattr(tup, _stfld("st_mtime"))) +
+ 1E-9 * getattr(tup, _stfld("nsec_mtime")))
+
+def _ll_get_st_ctime(tup):
+ return (float(getattr(tup, _stfld("st_ctime"))) +
+ 1E-9 * getattr(tup, _stfld("nsec_ctime")))
+
class __extend__(pairtype(StatResultRepr, IntegerRepr)):
def rtype_getitem((r_sta, r_int), hop):
s_int = hop.args_s[1]
index = s_int.const
+ if index < 0:
+ index += len(STAT_FIELDS)
return r_sta.redispatch_getfield(hop, index)
s_StatResult = SomeStatResult()
+
def make_stat_result(tup):
- """Turn a tuple into an os.stat_result object."""
- positional = tuple(
- lltype.cast_primitive(TYPE, value) for value, (name, TYPE) in
- zip(tup, STAT_FIELDS)[:N_INDEXABLE_FIELDS])
+ """NOT_RPYTHON: Turn a tuple into an os.stat_result object."""
+ assert len(tup) == len(STAT_FIELDS)
+ assert float not in [type(x) for x in tup]
+ positional = []
+ for i in range(N_INDEXABLE_FIELDS):
+ name, TYPE = STAT_FIELDS[i]
+ value = lltype.cast_primitive(TYPE, tup[i])
+ positional.append(value)
kwds = {}
+ kwds['st_atime'] = tup[7] + 1e-9 * tup[-3]
+ kwds['st_mtime'] = tup[8] + 1e-9 * tup[-2]
+ kwds['st_ctime'] = tup[9] + 1e-9 * tup[-1]
for value, (name, TYPE) in zip(tup, STAT_FIELDS)[N_INDEXABLE_FIELDS:]:
+ if name.startswith('nsec_'):
+ continue # ignore the nsec_Xtime here
kwds[name] = lltype.cast_primitive(TYPE, value)
return os.stat_result(positional, kwds)
@@ -360,6 +436,8 @@
posix_declaration(ALL_STAT_FIELDS[_i])
del _i
+STAT_FIELDS += ALL_STAT_FIELDS[-3:] # nsec_Xtime
+
# these two global vars only list the fields defined in the underlying platform
STAT_FIELD_TYPES = dict(STAT_FIELDS) # {'st_xxx': TYPE}
STAT_FIELD_NAMES = [_name for (_name, _TYPE) in STAT_FIELDS]
@@ -368,16 +446,20 @@
STATVFS_FIELD_TYPES = dict(STATVFS_FIELDS)
STATVFS_FIELD_NAMES = [name for name, tp in STATVFS_FIELDS]
+
def build_stat_result(st):
# only for LL backends
if TIMESPEC is not None:
- atim = st.c_st_atim; atime = int(atim.c_tv_sec) + 1E-9 *
int(atim.c_tv_nsec)
- mtim = st.c_st_mtim; mtime = int(mtim.c_tv_sec) + 1E-9 *
int(mtim.c_tv_nsec)
- ctim = st.c_st_ctim; ctime = int(ctim.c_tv_sec) + 1E-9 *
int(ctim.c_tv_nsec)
+ atim = st.c_st_atim
+ mtim = st.c_st_mtim
+ ctim = st.c_st_ctim
+ atime, extra_atime = atim.c_tv_sec, int(atim.c_tv_nsec)
+ mtime, extra_mtime = mtim.c_tv_sec, int(mtim.c_tv_nsec)
+ ctime, extra_ctime = ctim.c_tv_sec, int(ctim.c_tv_nsec)
else:
- atime = st.c_st_atime
- mtime = st.c_st_mtime
- ctime = st.c_st_ctime
+ atime, extra_atime = st.c_st_atime, 0
+ mtime, extra_mtime = st.c_st_mtime, 0
+ ctime, extra_ctime = st.c_st_ctime, 0
result = (st.c_st_mode,
st.c_st_ino,
@@ -395,6 +477,10 @@
if "st_rdev" in STAT_FIELD_TYPES: result += (st.c_st_rdev,)
if "st_flags" in STAT_FIELD_TYPES: result += (st.c_st_flags,)
+ result += (extra_atime,
+ extra_mtime,
+ extra_ctime)
+
return make_stat_result(result)
@@ -455,12 +541,14 @@
# console or LPT device
return make_stat_result((win32traits._S_IFCHR,
0, 0, 0, 0, 0,
- 0, 0, 0, 0))
+ 0, 0, 0, 0,
+ 0, 0, 0))
elif filetype == win32traits.FILE_TYPE_PIPE:
# socket or named pipe
return make_stat_result((win32traits._S_IFIFO,
0, 0, 0, 0, 0,
- 0, 0, 0, 0))
+ 0, 0, 0, 0,
+ 0, 0, 0))
elif filetype == win32traits.FILE_TYPE_UNKNOWN:
error = rwin32.GetLastError_saved()
if error != 0:
@@ -539,14 +627,11 @@
#__________________________________________________
# Helper functions for win32
if _WIN32:
- from rpython.rlib.rwin32file import FILE_TIME_to_time_t_float
+ from rpython.rlib.rwin32file import FILE_TIME_to_time_t_nsec
def make_longlong(high, low):
return (rffi.r_longlong(high) << 32) + rffi.r_longlong(low)
- # Seconds between 1.1.1601 and 1.1.1970
- secs_between_epochs = rffi.r_longlong(11644473600)
-
@specialize.arg(0)
def win32_xstat(traits, path, traverse=False):
win32traits = make_win32_traits(traits)
@@ -582,14 +667,15 @@
def win32_attribute_data_to_stat(win32traits, info):
st_mode = win32_attributes_to_mode(win32traits,
info.c_dwFileAttributes)
st_size = make_longlong(info.c_nFileSizeHigh, info.c_nFileSizeLow)
- ctime = FILE_TIME_to_time_t_float(info.c_ftCreationTime)
- mtime = FILE_TIME_to_time_t_float(info.c_ftLastWriteTime)
- atime = FILE_TIME_to_time_t_float(info.c_ftLastAccessTime)
+ ctime, extra_ctime = FILE_TIME_to_time_t_nsec(info.c_ftCreationTime)
+ mtime, extra_mtime = FILE_TIME_to_time_t_nsec(info.c_ftLastWriteTime)
+ atime, extra_atime = FILE_TIME_to_time_t_nsec(info.c_ftLastAccessTime)
result = (st_mode,
0, 0, 0, 0, 0,
st_size,
- atime, mtime, ctime)
+ atime, mtime, ctime,
+ extra_atime, extra_mtime, extra_ctime)
return make_stat_result(result)
@@ -597,9 +683,9 @@
# similar to the one above
st_mode = win32_attributes_to_mode(win32traits,
info.c_dwFileAttributes)
st_size = make_longlong(info.c_nFileSizeHigh, info.c_nFileSizeLow)
- ctime = FILE_TIME_to_time_t_float(info.c_ftCreationTime)
- mtime = FILE_TIME_to_time_t_float(info.c_ftLastWriteTime)
- atime = FILE_TIME_to_time_t_float(info.c_ftLastAccessTime)
+ ctime, extra_ctime = FILE_TIME_to_time_t_nsec(info.c_ftCreationTime)
+ mtime, extra_mtime = FILE_TIME_to_time_t_nsec(info.c_ftLastWriteTime)
+ atime, extra_atime = FILE_TIME_to_time_t_nsec(info.c_ftLastAccessTime)
# specific to fstat()
st_ino = make_longlong(info.c_nFileIndexHigh, info.c_nFileIndexLow)
@@ -608,7 +694,8 @@
result = (st_mode,
st_ino, 0, st_nlink, 0, 0,
st_size,
- atime, mtime, ctime)
+ atime, mtime, ctime,
+ extra_atime, extra_mtime, extra_ctime)
return make_stat_result(result)
diff --git a/rpython/rlib/rwin32file.py b/rpython/rlib/rwin32file.py
--- a/rpython/rlib/rwin32file.py
+++ b/rpython/rlib/rwin32file.py
@@ -6,6 +6,7 @@
from rpython.translator.tool.cbuild import ExternalCompilationInfo
from rpython.rtyper.tool import rffi_platform as platform
from rpython.rlib.objectmodel import specialize
+from rpython.rlib.rarithmetic import intmask
@specialize.memo()
def make_win32_traits(traits):
@@ -213,13 +214,25 @@
return (rffi.r_longlong(high) << 32) + rffi.r_longlong(low)
# Seconds between 1.1.1601 and 1.1.1970
-secs_between_epochs = rffi.r_longlong(11644473600)
+secs_between_epochs = 11644473600.0
+hns_between_epochs = rffi.r_longlong(116444736000000000) # units of 100 nsec
def FILE_TIME_to_time_t_float(filetime):
ft = make_longlong(filetime.c_dwHighDateTime, filetime.c_dwLowDateTime)
# FILETIME is in units of 100 nsec
return float(ft) * (1.0 / 10000000.0) - secs_between_epochs
+def FILE_TIME_to_time_t_nsec(filetime):
+ """Like the previous function, but returns a pair: (integer part
+ 'time_t' as a r_longlong, fractional part as an int measured in
+ nanoseconds).
+ """
+ ft = make_longlong(filetime.c_dwHighDateTime, filetime.c_dwLowDateTime)
+ ft -= hns_between_epochs
+ int_part = ft / 10000000
+ frac_part = ft - (int_part * 10000000)
+ return (int_part, intmask(frac_part) * 100)
+
def time_t_to_FILE_TIME(time, filetime):
ft = rffi.r_longlong((time + secs_between_epochs) * 10000000)
filetime.c_dwHighDateTime = rffi.r_uint(ft >> 32)
diff --git a/rpython/rlib/test/test_rposix_stat.py
b/rpython/rlib/test/test_rposix_stat.py
--- a/rpython/rlib/test/test_rposix_stat.py
+++ b/rpython/rlib/test/test_rposix_stat.py
@@ -2,12 +2,20 @@
import py
from rpython.rlib import rposix_stat
from rpython.tool.udir import udir
+from rpython.translator.c.test.test_genc import compile
+from rpython.rtyper.lltypesystem import lltype
+
class TestPosixStatFunctions:
@py.test.mark.skipif("sys.platform == 'win32'",
reason="win32 only has the portable fields")
def test_has_all_fields(self):
- assert rposix_stat.STAT_FIELDS == rposix_stat.ALL_STAT_FIELDS[:13]
+ # XXX this test is obscure! it will fail if the exact set of
+ # XXX stat fields found differs from the one we expect on Linux.
+ # XXX Why?
+ assert rposix_stat.STAT_FIELDS == (
+ rposix_stat.ALL_STAT_FIELDS[:13] +
+ rposix_stat.ALL_STAT_FIELDS[-3:])
def test_stat(self):
def check(f):
@@ -66,3 +74,27 @@
finally:
os.close(dirfd)
assert result.st_atime == tmpdir.join('file').atime()
+
+def test_high_precision_stat_time():
+ def f():
+ st = os.stat('.')
+ # should be supported on all platforms, but give a result whose
+ # precision might be lower than full nanosecond
+ highprec = rposix_stat.get_stat_ns_as_bigint(st, "ctime")
+ return '%s;%s' % (st.st_ctime, highprec.str())
+ fc = compile(f, [])
+ as_string = fc()
+ asfloat, highprec = as_string.split(';')
+ asfloat = float(asfloat)
+ highprec = int(highprec)
+ st = os.stat('.')
+ assert abs(asfloat - st.st_ctime) < 500e-9
+ assert abs(highprec - int(st.st_ctime * 1e9)) < 500
+ assert abs(rposix_stat.get_stat_ns_as_bigint(st, "ctime").tolong()
+ - st.st_ctime * 1e9) < 3
+ if rposix_stat.TIMESPEC is not None:
+ with lltype.scoped_alloc(rposix_stat.STAT_STRUCT.TO) as stresult:
+ rposix_stat.c_stat(".", stresult)
+ assert 0 <= stresult.c_st_ctim.c_tv_nsec <= 999999999
+ assert highprec == (int(stresult.c_st_ctim.c_tv_sec) * 1000000000
+ + int(stresult.c_st_ctim.c_tv_nsec))
_______________________________________________
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit
