mirror of
https://github.com/SDL-Hercules-390/hyperion.git
synced 2026-08-04 15:49:13 +02:00
HTHREADS/PTTRACE rework Part 2(b): Report abandoned locks
Still to be done: Part 3(a): fthreads R/W locks support
Part 3(b): Robust Mutexes support
This commit is contained in:
@@ -119,6 +119,7 @@ static CMDTAB cmdtab[] = { /* (COMMAND table) */
|
||||
CALL_EXTCMD ( cachestats_cmd ) /* (lives in cache.c) */
|
||||
CALL_EXTCMD ( shared_cmd ) /* (lives in shared.c) */
|
||||
CALL_EXTCMD ( ptt_cmd ) /* (lives in pttrace.c) */
|
||||
CALL_EXTCMD ( locks_cmd ) /* (lives in hthreads.c) */
|
||||
|
||||
/*-------------------------------------------------------------------*/
|
||||
/* $zapcmd - internal debug - may cause havoc - use with caution */
|
||||
|
||||
@@ -666,6 +666,11 @@
|
||||
"identical to the 'loadcore' command except that it loads a text deck\n" \
|
||||
"file with \"TXT\" and \"END\" 80 byte records (i.e. an object deck).\n"
|
||||
|
||||
#define locks_cmd_desc "Display internal locks list"
|
||||
#define locks_cmd_help \
|
||||
\
|
||||
"Format: \"locks [HELD|tid|ALL] [SORT [TIME|TOD]|[OWNER|TID]|NAME|LOC]\"\n"
|
||||
|
||||
#define log_cmd_desc "Direct logger output"
|
||||
#define log_cmd_help \
|
||||
\
|
||||
@@ -1594,6 +1599,8 @@ COMMAND( "exit", quit_cmd, SYSALLNDIAG8,
|
||||
|
||||
COMMAND( "sizeof", sizeof_cmd, SYSCMDNOPERNPROG, sizeof_cmd_desc, NULL )
|
||||
|
||||
COMMAND( "locks", EXTCMD( locks_cmd ), SYSPROGDEVEL, locks_cmd_desc, locks_cmd_help )
|
||||
|
||||
/*-------------------------------------------------------------------*/
|
||||
/* Commands optional by build option */
|
||||
/*-------------------------------------------------------------------*/
|
||||
|
||||
@@ -105,6 +105,15 @@ static void* test_thread( void* parg)
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* $test command helper thread */
|
||||
static void* test_locks_thread( void* parg)
|
||||
{
|
||||
// test thread exit with lock still held
|
||||
static LOCK testlock;
|
||||
initialize_lock( &testlock );
|
||||
obtain_lock( &testlock );
|
||||
return NULL;
|
||||
}
|
||||
|
||||
#define NUM_THREADS 10
|
||||
#define MAX_WAIT_SECS 6
|
||||
@@ -119,8 +128,18 @@ int test_cmd(int argc, char *argv[],char *cmdline)
|
||||
UNREFERENCED(cmdline);
|
||||
|
||||
if (argc > 1)
|
||||
{
|
||||
if ( CMD(argv[1],crash,5) )
|
||||
cause_crash(); // (see hscutl.c)
|
||||
else if (CMD( argv[1], locks, 5 ))
|
||||
{
|
||||
// test thread exit with lock still held
|
||||
static TID tid;
|
||||
VERIFY( create_thread( &tid, DETACHED,
|
||||
test_locks_thread, 0, "test_locks_thread" ) == 0);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/*-------------------------------------------*/
|
||||
/* test 'nanosleep' */
|
||||
|
||||
+606
-198
File diff suppressed because it is too large
Load Diff
+38
-42
@@ -212,75 +212,71 @@ typedef pthread_rwlock_t HRWLOCK;
|
||||
#endif /* !defined( OPTION_FTHREADS ) */
|
||||
|
||||
/*-------------------------------------------------------------------*/
|
||||
/* hthreads lock structures */
|
||||
/* Hercules lock structures */
|
||||
/*-------------------------------------------------------------------*/
|
||||
struct LOCK
|
||||
{
|
||||
TID tid; /* Thread-Id of who obtained lock */
|
||||
const char* loc; /* Location where it was obtained */
|
||||
HLOCK locklock; /* Internal LOCK structure lock */
|
||||
HLOCK lock; /* The actual locking model lock */
|
||||
void* ilk; /* ptr to internal ILOCK structure */
|
||||
};
|
||||
typedef struct LOCK LOCK;
|
||||
|
||||
struct RWLOCK
|
||||
{
|
||||
TID tid; /* Thread-Id of who obtained lock */
|
||||
const char* loc; /* Location where it was obtained */
|
||||
HLOCK locklock; /* Internal LOCK structure lock */
|
||||
HRWLOCK lock; /* The actual locking model rwlock */
|
||||
void* ilk; /* ptr to internal ILOCK structure */
|
||||
};
|
||||
typedef struct RWLOCK RWLOCK;
|
||||
|
||||
/*-------------------------------------------------------------------*/
|
||||
/* hthreads exported functions */
|
||||
/*-------------------------------------------------------------------*/
|
||||
HT_DLL_IMPORT int hthread_initialize_lock ( LOCK* plk, const char* loc );
|
||||
HT_DLL_IMPORT int hthread_obtain_lock ( LOCK* plk, const char* loc );
|
||||
HT_DLL_IMPORT int hthread_try_obtain_lock ( LOCK* plk, const char* loc );
|
||||
HT_DLL_IMPORT int hthread_test_lock ( LOCK* plk, const char* loc );
|
||||
HT_DLL_IMPORT int hthread_release_lock ( LOCK* plk, const char* loc );
|
||||
HT_DLL_IMPORT int hthread_destroy_lock ( LOCK* plk, const char* loc );
|
||||
HT_DLL_IMPORT int locks_cmd( int argc, char* argv[], char* cmdline );
|
||||
|
||||
HT_DLL_IMPORT int hthread_initialize_rwlock ( RWLOCK* plk, const char* loc );
|
||||
HT_DLL_IMPORT int hthread_destroy_rwlock ( RWLOCK* plk, const char* loc );
|
||||
HT_DLL_IMPORT int hthread_obtain_rdlock ( RWLOCK* plk, const char* loc );
|
||||
HT_DLL_IMPORT int hthread_obtain_wrlock ( RWLOCK* plk, const char* loc );
|
||||
HT_DLL_IMPORT int hthread_release_rwlock ( RWLOCK* plk, const char* loc );
|
||||
HT_DLL_IMPORT int hthread_try_obtain_rdlock ( RWLOCK* plk, const char* loc );
|
||||
HT_DLL_IMPORT int hthread_try_obtain_wrlock ( RWLOCK* plk, const char* loc );
|
||||
HT_DLL_IMPORT int hthread_test_rdlock ( RWLOCK* plk, const char* loc );
|
||||
HT_DLL_IMPORT int hthread_test_wrlock ( RWLOCK* plk, const char* loc );
|
||||
HT_DLL_IMPORT int hthread_initialize_lock ( LOCK* plk, const char* name, const char* location );
|
||||
HT_DLL_IMPORT int hthread_obtain_lock ( LOCK* plk, const char* location );
|
||||
HT_DLL_IMPORT int hthread_try_obtain_lock ( LOCK* plk, const char* location );
|
||||
HT_DLL_IMPORT int hthread_test_lock ( LOCK* plk, const char* location );
|
||||
HT_DLL_IMPORT int hthread_release_lock ( LOCK* plk, const char* location );
|
||||
HT_DLL_IMPORT int hthread_destroy_lock ( LOCK* plk, const char* location );
|
||||
|
||||
HT_DLL_IMPORT int hthread_initialize_condition ( COND* plc, const char* loc );
|
||||
HT_DLL_IMPORT int hthread_destroy_condition ( COND* plc, const char* loc );
|
||||
HT_DLL_IMPORT int hthread_signal_condition ( COND* plc, const char* loc );
|
||||
HT_DLL_IMPORT int hthread_broadcast_condition ( COND* plc, const char* loc );
|
||||
HT_DLL_IMPORT int hthread_wait_condition ( COND* plc, LOCK* plk, const char* loc );
|
||||
HT_DLL_IMPORT int hthread_timed_wait_condition ( COND* plc, LOCK* plk, const struct timespec* tm, const char* loc );
|
||||
HT_DLL_IMPORT int hthread_initialize_rwlock ( RWLOCK* plk, const char* name, const char* location );
|
||||
HT_DLL_IMPORT int hthread_destroy_rwlock ( RWLOCK* plk, const char* location );
|
||||
HT_DLL_IMPORT int hthread_obtain_rdlock ( RWLOCK* plk, const char* location );
|
||||
HT_DLL_IMPORT int hthread_obtain_wrlock ( RWLOCK* plk, const char* location );
|
||||
HT_DLL_IMPORT int hthread_release_rwlock ( RWLOCK* plk, const char* location );
|
||||
HT_DLL_IMPORT int hthread_try_obtain_rdlock ( RWLOCK* plk, const char* location );
|
||||
HT_DLL_IMPORT int hthread_try_obtain_wrlock ( RWLOCK* plk, const char* location );
|
||||
HT_DLL_IMPORT int hthread_test_rdlock ( RWLOCK* plk, const char* location );
|
||||
HT_DLL_IMPORT int hthread_test_wrlock ( RWLOCK* plk, const char* location );
|
||||
|
||||
HT_DLL_IMPORT int hthread_initialize_join_attr ( ATTR* pat, const char* loc );
|
||||
HT_DLL_IMPORT int hthread_initialize_detach_attr ( ATTR* pat, const char* loc );
|
||||
HT_DLL_IMPORT int hthread_create_thread ( TID* ptid, ATTR* pat, THREAD_FUNC* pfn, void* arg, const char* name, const char* loc );
|
||||
HT_DLL_IMPORT int hthread_join_thread ( TID tid, void** prc, const char* loc );
|
||||
HT_DLL_IMPORT int hthread_detach_thread ( TID tid, const char* loc );
|
||||
HT_DLL_IMPORT int hthread_signal_thread ( TID tid, int sig, const char* loc );
|
||||
HT_DLL_IMPORT TID hthread_thread_id ( const char* loc );
|
||||
HT_DLL_IMPORT void hthread_exit_thread ( void* rc, const char* loc );
|
||||
HT_DLL_IMPORT int hthread_equal_threads ( TID tid1, TID tid2, const char* loc );
|
||||
HT_DLL_IMPORT int hthread_win_thread_handle ( TID tid, const char* loc );
|
||||
HT_DLL_IMPORT int hthread_initialize_condition ( COND* plc, const char* location );
|
||||
HT_DLL_IMPORT int hthread_destroy_condition ( COND* plc, const char* location );
|
||||
HT_DLL_IMPORT int hthread_signal_condition ( COND* plc, const char* location );
|
||||
HT_DLL_IMPORT int hthread_broadcast_condition ( COND* plc, const char* location );
|
||||
HT_DLL_IMPORT int hthread_wait_condition ( COND* plc, LOCK* plk, const char* location );
|
||||
HT_DLL_IMPORT int hthread_timed_wait_condition ( COND* plc, LOCK* plk, const struct timespec* tm, const char* location );
|
||||
|
||||
HT_DLL_IMPORT int hthread_initialize_join_attr ( ATTR* pat, const char* location );
|
||||
HT_DLL_IMPORT int hthread_initialize_detach_attr ( ATTR* pat, const char* location );
|
||||
HT_DLL_IMPORT int hthread_create_thread ( TID* ptid, ATTR* pat, THREAD_FUNC* pfn, void* arg, const char* name, const char* location );
|
||||
HT_DLL_IMPORT int hthread_join_thread ( TID tid, void** prc, const char* location );
|
||||
HT_DLL_IMPORT int hthread_detach_thread ( TID tid, const char* location );
|
||||
HT_DLL_IMPORT int hthread_signal_thread ( TID tid, int sig, const char* location );
|
||||
HT_DLL_IMPORT TID hthread_thread_id ( const char* location );
|
||||
HT_DLL_IMPORT void hthread_exit_thread ( void* rc, const char* location );
|
||||
HT_DLL_IMPORT int hthread_equal_threads ( TID tid1, TID tid2, const char* location );
|
||||
HT_DLL_IMPORT int hthread_win_thread_handle ( TID tid, const char* location );
|
||||
|
||||
/*-------------------------------------------------------------------*/
|
||||
/* Hercules threading/locking macros */
|
||||
/*-------------------------------------------------------------------*/
|
||||
#define initialize_lock( plk ) hthread_initialize_lock( plk, PTT_LOC )
|
||||
#define initialize_lock( plk ) hthread_initialize_lock( plk, #plk, PTT_LOC )
|
||||
#define obtain_lock( plk ) hthread_obtain_lock( plk, PTT_LOC )
|
||||
#define try_obtain_lock( plk ) hthread_try_obtain_lock( plk, PTT_LOC )
|
||||
#define test_lock( plk ) hthread_test_lock( plk, PTT_LOC )
|
||||
#define release_lock(plk) hthread_release_lock( plk, PTT_LOC )
|
||||
#define destroy_lock(plk) hthread_destroy_lock( plk, PTT_LOC )
|
||||
|
||||
#define initialize_rwlock( plk ) hthread_initialize_rwlock( plk, PTT_LOC )
|
||||
#define initialize_rwlock( plk ) hthread_initialize_rwlock( plk, #plk, PTT_LOC )
|
||||
#define destroy_rwlock( plk ) hthread_destroy_rwlock( plk, PTT_LOC )
|
||||
#define obtain_rdlock( plk ) hthread_obtain_rdlock( plk, PTT_LOC )
|
||||
#define obtain_wrlock( plk ) hthread_obtain_wrlock( plk, PTT_LOC )
|
||||
|
||||
@@ -1999,12 +1999,17 @@ do { \
|
||||
#define HHC90000 "DBG: %s"
|
||||
#define HHC90001 " *** Assertion Failed! *** %s(%d); function: %s()"
|
||||
|
||||
/* pttrace.c */
|
||||
/* hthreads.c, pttrace.c */
|
||||
#define HHC90010 "Pttrace: trace is busy"
|
||||
#define HHC90011 "Pttrace: invalid argument %s"
|
||||
#define HHC90012 "Pttrace: %s%s%s%s%s%s%s%s%s%s%s %s %s to %d %d"
|
||||
#define HHC90013 "'%s' failed: rc=%d (%s), tid="TIDPAT", loc=%s"
|
||||
#define HHC90014 "lock was obtained by thread "TIDPAT" at %s"
|
||||
#define HHC90013 "'%s(%s)' failed: rc=%d: %s; tid="TIDPAT", loc=%s"
|
||||
#define HHC90014 "lock %s was obtained by thread "TIDPAT" at %s"
|
||||
#define HHC90015 "Thread "TIDPAT" abandoned lock %s obtained on %s at %s"
|
||||
#define HHC90016 "Thread "TIDPAT" abandoned at %s lock %s obtained on %s at %s"
|
||||
#define HHC90017 "Lock=%s, tid="TIDPAT", tod=%s, loc=%s"
|
||||
#define HHC90018 "Total locks defined: %d"
|
||||
#define HHC90019 "No locks found for thread "TIDPAT"."
|
||||
|
||||
/* from crypto/dyncrypt.c when compiled with debug on */
|
||||
#define HHC90100 "%s"
|
||||
|
||||
@@ -381,7 +381,7 @@ DLL_EXPORT void ptt_trace_init( int n, int init )
|
||||
/*-------------------------------------------------------------------*/
|
||||
DLL_EXPORT void ptt_pthread_trace (int trclass, const char *msg,
|
||||
const void *data1, const void *data2,
|
||||
const char *loc, int rc)
|
||||
const char *loc, int rc, TIMEVAL* pTV)
|
||||
{
|
||||
int i, n;
|
||||
|
||||
@@ -413,7 +413,12 @@ int i, n;
|
||||
|
||||
/* Fill in the trace table entry */
|
||||
if (pttnotod == 0)
|
||||
gettimeofday(&pttrace[i].tv,NULL);
|
||||
{
|
||||
if (pTV)
|
||||
memcpy( &pttrace[i].tv, pTV, sizeof( TIMEVAL ));
|
||||
else
|
||||
gettimeofday( &pttrace[i].tv, NULL );
|
||||
}
|
||||
pttrace[i].tid = hthread_self();
|
||||
pttrace[i].trclass = trclass;
|
||||
pttrace[i].msg = msg;
|
||||
@@ -430,8 +435,7 @@ DLL_EXPORT int ptt_pthread_print ()
|
||||
{
|
||||
int i, n, count = 0;
|
||||
char retcode[32]; // (retcode is 'int'; if x64, 19 digits or more!)
|
||||
char tbuf[256];
|
||||
time_t tt;
|
||||
char tod[27]; // "YYYY-MM-DD HH:MM:SS.uuuuuu"
|
||||
|
||||
if (pttrace && pttracen)
|
||||
{
|
||||
@@ -447,7 +451,7 @@ time_t tt;
|
||||
{
|
||||
if (pttrace[i].tid)
|
||||
{
|
||||
tt = pttrace[i].tv.tv_sec; strlcpy(tbuf, ctime(&tt),sizeof(tbuf)); tbuf[19] = '\0';
|
||||
FormatTIMEVAL( &pttrace[i].tv, tod, sizeof( tod ));
|
||||
|
||||
if (pttrace[i].rc == PTT_MAGIC && (pttrace[i].trclass & PTT_CL_THR))
|
||||
retcode[0] = '\0';
|
||||
@@ -459,7 +463,7 @@ time_t tt;
|
||||
logmsg
|
||||
(
|
||||
"%-18s " // File name
|
||||
"%s.%6.6ld " // Time of day (HH:MM:SS.usecs)
|
||||
"%s " // Time of day (HH:MM:SS.usecs)
|
||||
I32_FMTX" " // Thread id (low 32 bits)
|
||||
"%-12s " // Trace message (string; 12 chars)
|
||||
PTR_FMTx" " // Data value 1
|
||||
@@ -467,8 +471,7 @@ time_t tt;
|
||||
"%s\n" // Return code (or empty string)
|
||||
|
||||
,pttrace[i].loc // File name
|
||||
,tbuf + 11 // Time of day (HH:MM:SS)
|
||||
,pttrace[i].tv.tv_usec // Time of day (usecs)
|
||||
,&tod[11] // Time of day (HH:MM:SS.usecs)
|
||||
,(U32)(uintptr_t)(pttrace[i].tid) // Thread id (low 32 bits)
|
||||
,pttrace[i].msg // Trace message (string; 12 chars)
|
||||
,(uintptr_t)pttrace[i].data1 // Data value 1
|
||||
|
||||
@@ -70,7 +70,7 @@ do { \
|
||||
ptt_pthread_trace( (_class), (_msg),(void*)(uintptr_t)(_data1), \
|
||||
(void*)(uintptr_t)(_data2), \
|
||||
PTT_LOC, \
|
||||
(int)(_rc)); \
|
||||
(int)(_rc),NULL); \
|
||||
} while(0)
|
||||
|
||||
/*-------------------------------------------------------------------*/
|
||||
@@ -85,7 +85,7 @@ do { \
|
||||
/*-------------------------------------------------------------------*/
|
||||
PTT_DLL_IMPORT void ptt_trace_init ( int n, int init );
|
||||
PTT_DLL_IMPORT int ptt_cmd ( int argc, char* argv[], char* cmdline );
|
||||
PTT_DLL_IMPORT void ptt_pthread_trace ( int, const char*, const void*, const void*, const char*, int );
|
||||
PTT_DLL_IMPORT void ptt_pthread_trace ( int, const char*, const void*, const void*, const char*, int, TIMEVAL* );
|
||||
PTT_DLL_IMPORT int ptt_pthread_print ();
|
||||
PTT_DLL_IMPORT U32 pttclass;
|
||||
PTT_DLL_IMPORT int pttthread;
|
||||
|
||||
@@ -443,8 +443,17 @@ static ULARGE_INTEGER FileTimeTo1970Nanoseconds( const FILETIME* pFT )
|
||||
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////
|
||||
// (PUBLIC) Nanosecond resolution TOD clock (clock_gettime)
|
||||
|
||||
// (PUBLIC) Nanosecond resolution (not quite but almost!) TOD clock (clock_gettime)
|
||||
//
|
||||
// ** CRITICAL PROGRAMMING NOTE! **
|
||||
//
|
||||
// Because the new hthreads design calls gettimeofday to save the time when a lock is
|
||||
// initialized or obtained, etc, the below function nor any of the functions it calls
|
||||
// may call logmsg either directly or indirectly (such as using the 'TRACE' macro) or
|
||||
// else an infinite loop will occur since our logger design uses locks! hthreads will
|
||||
// call gettimeofday which issues a message which calls logger which uses a lock and
|
||||
// hthreads calls gettimeofday again, etc.
|
||||
//
|
||||
DLL_EXPORT int clock_gettime ( clockid_t clk_id, struct timespec *tp )
|
||||
{
|
||||
ULARGE_INTEGER uliWork; // (current HPC tick count and work)
|
||||
@@ -494,7 +503,6 @@ DLL_EXPORT int clock_gettime ( clockid_t clk_id, struct timespec *tp )
|
||||
if (!uliHPCTicksPerSec.QuadPart) // (we only need to do this once)
|
||||
{
|
||||
VERIFY( QueryPerformanceFrequency( (LARGE_INTEGER*)&uliHPCTicksPerSec ));
|
||||
TRACE("w32util: uliHPCTicksPerSec = 0x%16.16llX (%llu)\n", uliHPCTicksPerSec.QuadPart, uliHPCTicksPerSec.QuadPart );
|
||||
|
||||
// Verify the length of time between host TOD clock resyncs isn't
|
||||
// so very long that the number of High Performance Counter ticks
|
||||
@@ -503,11 +511,9 @@ DLL_EXPORT int clock_gettime ( clockid_t clk_id, struct timespec *tp )
|
||||
|
||||
while (uliHPCTicksPerSec.QuadPart > (_UI64_MAX / (uiResyncSecs + 1)))
|
||||
uiResyncSecs--;
|
||||
TRACE("w32util: uiResyncSecs = %lu\n", uiResyncSecs );
|
||||
|
||||
uliMaxElapsedHPCTicks.QuadPart =
|
||||
uliHPCTicksPerSec.QuadPart * uiResyncSecs;
|
||||
TRACE("w32util: uliMaxElapsedHPCTicks = 0x%16.16llX (%llu)\n", uliMaxElapsedHPCTicks.QuadPart, uliMaxElapsedHPCTicks.QuadPart );
|
||||
|
||||
// Calculate the maximum supported clock resolution such that we don't
|
||||
// resync with the host TOD clock more than once every resync interval.
|
||||
@@ -517,7 +523,6 @@ DLL_EXPORT int clock_gettime ( clockid_t clk_id, struct timespec *tp )
|
||||
{
|
||||
u64ClockResolution /= 10; // (decrease TOD clock resolution)
|
||||
}
|
||||
TRACE("w32util: u64ClockResolution = %llu (%11.9f)\n", u64ClockResolution, 1.0 / (double)u64ClockResolution );
|
||||
|
||||
// (check for error condition...)
|
||||
|
||||
@@ -529,7 +534,6 @@ DLL_EXPORT int clock_gettime ( clockid_t clk_id, struct timespec *tp )
|
||||
}
|
||||
|
||||
u64ClockNanoScale = (MAX_GTOD_RESOLUTION / u64ClockResolution);
|
||||
TRACE("w32util: u64ClockNanoScale = %llu\n", u64ClockNanoScale );
|
||||
}
|
||||
|
||||
uliStartingNanoTime = FileTimeTo1970Nanoseconds( &ftStartingSystemTime );
|
||||
@@ -623,7 +627,16 @@ DLL_EXPORT int clock_gettime ( clockid_t clk_id, struct timespec *tp )
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////
|
||||
// (PUBLIC) Microsecond resolution GTOD (getimeofday)...
|
||||
|
||||
//
|
||||
// ** CRITICAL PROGRAMMING NOTE! **
|
||||
//
|
||||
// Because the new hthreads design calls gettimeofday to save the time when a lock is
|
||||
// initialized or obtained, etc, the below function nor any of the functions it calls
|
||||
// may call logmsg either directly or indirectly (such as using the 'TRACE' macro) or
|
||||
// else an infinite loop will occur since our logger design uses locks! hthreads will
|
||||
// call gettimeofday which issues a message which calls logger which uses a lock and
|
||||
// hthreads calls gettimeofday again, etc.
|
||||
//
|
||||
DLL_EXPORT int gettimeofday ( struct timeval* pTV, void* pTZ )
|
||||
{
|
||||
static struct timeval tvPrevRetVal = {0};
|
||||
|
||||
Reference in New Issue
Block a user